diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 274773169407..280fccb7906f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,7 +6,7 @@ on: pull_request: # run on all PRs, not just PRs to a particular branch jobs: - # `basics` includes all non-smoke CI, except for unit and proto + # `basics` includes all non-smoke and non-unit CI basics: runs-on: ubuntu-latest @@ -78,12 +78,13 @@ jobs: matrix: os: [ubuntu-latest, windows-latest] runs-on: ${{ matrix.os }} - name: unit_${{ matrix.os }} + name: unit - ${{ matrix.os == 'ubuntu-latest' && 'Ubuntu' || 'Windows' }} steps: - name: git clone uses: actions/checkout@v2 with: + # Depth of at least 2 for codecov coverage diffs. See https://github.com/GoogleChrome/lighthouse/pull/12079 fetch-depth: 2 - name: Use Node.js 12.x @@ -134,100 +135,3 @@ jobs: # Fail if any changes were written to any source files or generated untracked files (ex, from -GA). - run: git add -A && git diff --cached --exit-code - - # `smoke` runs as a matrix across 4 jobs: - # * The smoketest groups are split across two runners, to parallelize. - # * Then, those are run with both Chrome stable and ToT Chromium, in parallel - smoke: - strategy: - matrix: - chrome-channel: ['stable', 'ToT'] - smoke-test-invert: [false, true] - # e.g. if smoke 0 fails, continue with smoke 1 anyway - fail-fast: false - runs-on: ubuntu-latest - env: - # The smokehouse tests run by job `smoke_0`. `smoke_1` will run the rest. - SMOKE_GROUP_1: a11y oopif pwa pwa2 pwa3 dbw redirects errors offline - name: smoke_${{ strategy.job-index }}_${{ matrix.chrome-channel }} - - steps: - - name: git clone - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - - name: Use Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: 12.x - - - name: Define ToT chrome path - if: matrix.chrome-channel == 'ToT' - run: echo "CHROME_PATH=/home/runner/chrome-linux-tot/chrome" >> $GITHUB_ENV - - # Chrome Stable is already installed by default. - - name: Install Chrome ToT - if: matrix.chrome-channel == 'ToT' - working-directory: /home/runner - run: bash $GITHUB_WORKSPACE/lighthouse-core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot - - - run: yarn install --frozen-lockfile --network-timeout 1000000 - - - run: sudo apt-get install xvfb - - name: Run smoke tests - run: | - xvfb-run --auto-servernum yarn c8 yarn smoke --debug -j=1 --retries=2 --invert-match ${{ matrix.smoke-test-invert }} $SMOKE_GROUP_1 - yarn c8 report --reporter text-lcov > smoke-coverage.lcov - - - name: Upload test coverage to Codecov - if: matrix.chrome-channel == 'ToT' - uses: codecov/codecov-action@6004246f47ab62d32be025ce173b241cd84ac58e - with: - flags: smoke - file: ./smoke-coverage.lcov - - # Fail if any changes were written to source files. - - run: git diff --exit-code - - # Only run smoke tests for windows against stable chrome. - smoke-windows: - runs-on: windows-latest - name: smoke_stable_windows - - steps: - - name: git clone - uses: actions/checkout@v2 - - - name: Use Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: 12.x - - - run: yarn install --frozen-lockfile --network-timeout 1000000 - - - name: Run smoke tests - run: yarn smoke --debug -j=1 --retries=2 dbw oopif offline lantern metrics - - smoke-bundle: - runs-on: ubuntu-latest - name: smoke_bundle - - steps: - - name: git clone - uses: actions/checkout@v2 - - - name: Use Node.js 12.x - uses: actions/setup-node@v1 - with: - node-version: 12.x - - - run: yarn install --frozen-lockfile --network-timeout 1000000 - - run: yarn build-devtools - - - run: sudo apt-get install xvfb - - name: yarn test-bundle - run: xvfb-run --auto-servernum yarn test-bundle - - # Fail if any changes were written to source files. - - run: git diff --exit-code diff --git a/.github/workflows/devtools.yml b/.github/workflows/devtools.yml index 1b95eced3fef..94b325d2f7c5 100644 --- a/.github/workflows/devtools.yml +++ b/.github/workflows/devtools.yml @@ -6,7 +6,7 @@ on: pull_request: # run on all PRs, not just PRs to a particular branch jobs: - smoke-devtools: + integration: runs-on: macos-latest # while macbots are much slower, linux reliably crashes running this steps: diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml new file mode 100644 index 000000000000..f7bf9370768a --- /dev/null +++ b/.github/workflows/smoke.yml @@ -0,0 +1,110 @@ +name: smoke + +on: + push: + branches: [master] + pull_request: # run on all PRs, not just PRs to a particular branch + +jobs: + # `smoke` runs as a matrix across 4 jobs: + # * The smoke tests are split into two batches, to parallelize. + # * Then, those are run with both Chrome stable and ToT Chromium, in parallel + smoke: + strategy: + matrix: + chrome-channel: ['stable', 'ToT'] + smoke-test-invert: [false, true] + # e.g. if set 1 fails, continue with set 2 anyway + fail-fast: false + runs-on: ubuntu-latest + env: + # The smokehouse tests run by job `test set 1`. `test set 2` will run the rest. + SMOKE_BATCH_1: a11y oopif pwa pwa2 pwa3 dbw redirects errors offline + # Job named e.g. "Chrome stable, test set 1". + name: Chrome ${{ matrix.chrome-channel }}, batch ${{ matrix.smoke-test-invert == false && '1' || '2' }} + + steps: + - name: git clone + uses: actions/checkout@v2 + with: + # Depth of at least 2 for codecov coverage diffs. See https://github.com/GoogleChrome/lighthouse/pull/12079 + fetch-depth: 2 + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - name: Define ToT chrome path + if: matrix.chrome-channel == 'ToT' + run: echo "CHROME_PATH=/home/runner/chrome-linux-tot/chrome" >> $GITHUB_ENV + + # Chrome Stable is already installed by default. + - name: Install Chrome ToT + if: matrix.chrome-channel == 'ToT' + working-directory: /home/runner + run: bash $GITHUB_WORKSPACE/lighthouse-core/scripts/download-chrome.sh && mv chrome-linux chrome-linux-tot + + - run: yarn install --frozen-lockfile --network-timeout 1000000 + + - run: sudo apt-get install xvfb + - name: Run smoke tests + run: | + xvfb-run --auto-servernum yarn c8 yarn smoke --debug -j=1 --retries=2 --invert-match ${{ matrix.smoke-test-invert }} $SMOKE_BATCH_1 + yarn c8 report --reporter text-lcov > smoke-coverage.lcov + + - name: Upload test coverage to Codecov + if: matrix.chrome-channel == 'ToT' + uses: codecov/codecov-action@6004246f47ab62d32be025ce173b241cd84ac58e + with: + flags: smoke + file: ./smoke-coverage.lcov + + # Fail if any changes were written to source files. + - run: git diff --exit-code + + # Only run smoke tests for windows against stable chrome. + smoke-windows: + runs-on: windows-latest + name: Windows + + steps: + - name: git clone + uses: actions/checkout@v2 + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - run: yarn install --frozen-lockfile --network-timeout 1000000 + + - name: Run smoke tests + # Windows bots are slow, so only run enough tests to verify matching behavior. + run: yarn smoke --debug -j=1 --retries=2 dbw oopif offline lantern metrics + + # Fail if any changes were written to source files. + - run: git diff --exit-code + + smoke-bundle: + runs-on: ubuntu-latest + name: Bundled Lighthouse + + steps: + - name: git clone + uses: actions/checkout@v2 + + - name: Use Node.js 12.x + uses: actions/setup-node@v1 + with: + node-version: 12.x + + - run: yarn install --frozen-lockfile --network-timeout 1000000 + - run: yarn build-devtools + + - run: sudo apt-get install xvfb + - name: yarn test-bundle + run: xvfb-run --auto-servernum yarn test-bundle + + # Fail if any changes were written to source files. + - run: git diff --exit-code diff --git a/readme.md b/readme.md index 8745b958253d..dab3fe8d838e 100644 --- a/readme.md +++ b/readme.md @@ -1,4 +1,4 @@ -# Lighthouse [![GitHub Actions Status Badge](https://github.com/GoogleChrome/lighthouse/workflows/💡🏠/badge.svg)](https://github.com/GoogleChrome/lighthouse/actions) [![Coverage Status](https://codecov.io/gh/GoogleChrome/lighthouse/branch/master/graph/badge.svg)](https://codecov.io/gh/GoogleChrome/lighthouse) [![Build tracker for Lighthouse](https://img.shields.io/badge/buildtracker-ok-blue)](https://lh-build-tracker.herokuapp.com/) [![NPM lighthouse package](https://img.shields.io/npm/v/lighthouse.svg)](https://npmjs.org/package/lighthouse) +# Lighthouse [![GitHub Actions Status Badge](https://github.com/GoogleChrome/lighthouse/workflows/💡🏠/badge.svg)](https://github.com/GoogleChrome/lighthouse/actions/workflows/ci.yml) [![GitHub Actions Status Badge](https://github.com/GoogleChrome/lighthouse/workflows/smoke/badge.svg)](https://github.com/GoogleChrome/lighthouse/actions/workflows/smoke.yml) [![Coverage Status](https://codecov.io/gh/GoogleChrome/lighthouse/branch/master/graph/badge.svg)](https://codecov.io/gh/GoogleChrome/lighthouse) [![Build tracker for Lighthouse](https://img.shields.io/badge/buildtracker-ok-blue)](https://lh-build-tracker.herokuapp.com/) [![NPM lighthouse package](https://img.shields.io/npm/v/lighthouse.svg)](https://npmjs.org/package/lighthouse) > Lighthouse analyzes web apps and web pages, collecting modern performance metrics and insights on developer best practices.