.github/workflows/ci.yml

@@ -5,6 +5,9 @@ on:
     branches: [main]
   pull_request: # run on all PRs, not just PRs to a particular branch
 
+env:
+  PUPPETEER_SKIP_DOWNLOAD: 1
+
 jobs:
   # `basics` includes all non-smoke and non-unit CI
   basics:

.github/workflows/devtools.yml

@@ -8,6 +8,7 @@ on:
 env:
   DEPOT_TOOLS_PATH: ${{ github.workspace }}/depot-tools
   DEVTOOLS_PATH: ${{ github.workspace }}/devtools-frontend
+  PUPPETEER_SKIP_DOWNLOAD: 1
 
 jobs:
   build:
@@ -175,4 +176,4 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (devtools smoke)
-        path: ${{ github.workspace }}/lighthouse/.tmp/smokehouse-ci-failures/
+        path: ${{ github.workspace }}/lighthouse/.tmp/smokehouse-failures/

.github/workflows/package-test.yml

@@ -5,6 +5,9 @@ on:
     branches: [main]
   pull_request: # run on all PRs, not just PRs to a particular branch
 
+env:
+  PUPPETEER_SKIP_DOWNLOAD: 1
+
 jobs:
   package-test:
     runs-on: ubuntu-latest

.github/workflows/smoke.yml

@@ -5,6 +5,9 @@ on:
     branches: [main]
   pull_request: # run on all PRs, not just PRs to a particular branch
 
+env:
+  PUPPETEER_SKIP_DOWNLOAD: 1
+
 jobs:
   # `smoke` runs as a matrix across 6 jobs:
   #  * The smoke tests are split into 3 batches, to parallelize.
@@ -72,7 +75,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (ubuntu; chrome ${{ matrix.chrome-channel }})
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-windows:
     strategy:
@@ -115,7 +118,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (windows)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-legacy:
     strategy:
@@ -163,7 +166,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (legacy)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-bundle:
     strategy:
@@ -211,7 +214,7 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (bundled)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/
 
   smoke-bundle-legacy:
     strategy:
@@ -259,4 +262,4 @@ jobs:
       uses: actions/upload-artifact@v1
       with:
         name: Smokehouse (bundled)
-        path: .tmp/smokehouse-ci-failures/
+        path: .tmp/smokehouse-failures/

.github/workflows/unit.yml

@@ -5,6 +5,9 @@ on:
     branches: [main]
   pull_request: # run on all PRs, not just PRs to a particular branch
 
+env:
+  PUPPETEER_SKIP_DOWNLOAD: 1
+
 jobs:
   # `unit` includes just unit and proto tests.
   unit:

changelog.md

@@ -1,3 +1,67 @@
+<a name="10.4.0"></a>
+# 10.4.0 (2023-07-10)
+[Full Changelog](https://github.com/GoogleChrome/lighthouse/compare/v10.3.0...v10.4.0)
+
+We expect this release to ship in the DevTools of [Chrome 117](https://chromiumdash.appspot.com/schedule), and to PageSpeed Insights within 2 weeks.
+
+## New Contributors
+
+Thanks to our new contributors 👽🐷🐰🐯🐻!
+
+- Sanjaiyan Parthipan @sanjaiyan-dev
+
+## New Audits
+
+* add hidden, 0-weight a11y audits ([#15216](https://github.com/GoogleChrome/lighthouse/pull/15216))
+* aria-dialog-name, aria-text, link-in-text-block, select-name ([#15159](https://github.com/GoogleChrome/lighthouse/pull/15159))
+
+## Core
+
+* remove unnecessary references to legacy gatherer ([#15236](https://github.com/GoogleChrome/lighthouse/pull/15236))
+* update residual usages of legacy runner ([#15227](https://github.com/GoogleChrome/lighthouse/pull/15227))
+* compute TBT impact for main thread tasks ([#15175](https://github.com/GoogleChrome/lighthouse/pull/15175))
+* support ts targets before es2022 ([#15189](https://github.com/GoogleChrome/lighthouse/pull/15189))
+* byte-efficiency: compute FCP & LCP savings ([#15104](https://github.com/GoogleChrome/lighthouse/pull/15104))
+* config: add more validation from legacy ([#15211](https://github.com/GoogleChrome/lighthouse/pull/15211))
+* link-text: add tamil keywords to blocklist ([#15152](https://github.com/GoogleChrome/lighthouse/pull/15152))
+* long-tasks: add more task information to debugData ([#15198](https://github.com/GoogleChrome/lighthouse/pull/15198))
+* network-recorder: set target type of unfinished request ([#15232](https://github.com/GoogleChrome/lighthouse/pull/15232))
+* non-composited-animations: add CLS savings as always 0 ([#15099](https://github.com/GoogleChrome/lighthouse/pull/15099))
+* prioritize-lcp-image: add LCP savings ([#15229](https://github.com/GoogleChrome/lighthouse/pull/15229))
+* proto: add errorStack to AuditResult ([#15187](https://github.com/GoogleChrome/lighthouse/pull/15187))
+* redirects: add FCP and LCP savings ([#15228](https://github.com/GoogleChrome/lighthouse/pull/15228))
+* source-maps: support BOM markers and CORB prefix ([#15224](https://github.com/GoogleChrome/lighthouse/pull/15224))
+* stacks: add wix ([#15171](https://github.com/GoogleChrome/lighthouse/pull/15171))
+* unsized-images: add CLS savings as always 0 ([#15196](https://github.com/GoogleChrome/lighthouse/pull/15196))
+* viewport: add INP savings ([#15071](https://github.com/GoogleChrome/lighthouse/pull/15071))
+* work-during-interaction: add INP savings ([#15176](https://github.com/GoogleChrome/lighthouse/pull/15176))
+
+## Report
+
+* use fixed position for hidden radios ([#15181](https://github.com/GoogleChrome/lighthouse/pull/15181))
+
+## Deps
+
+* upgrade puppeteer to 20.8 ([#15226](https://github.com/GoogleChrome/lighthouse/pull/15226))
+* upgrade third-party-web to 0.23.3 ([#15213](https://github.com/GoogleChrome/lighthouse/pull/15213))
+
+## I18n
+
+* import ([#15243](https://github.com/GoogleChrome/lighthouse/pull/15243))
+
+## Tests
+
+* save smokehouse failures, improve bundle runner logging ([#15235](https://github.com/GoogleChrome/lighthouse/pull/15235))
+* remove usages of legacy driver ([#15230](https://github.com/GoogleChrome/lighthouse/pull/15230))
+* a11y: use regex for target size explanation ([#15231](https://github.com/GoogleChrome/lighthouse/pull/15231))
+* ci: skip puppeteer Chrome download ([#15177](https://github.com/GoogleChrome/lighthouse/pull/15177))
+* config: add unit tests from legacy config ([#15209](https://github.com/GoogleChrome/lighthouse/pull/15209))
+* devtools: fix e2e compile error ([#15210](https://github.com/GoogleChrome/lighthouse/pull/15210))
+* devtools: remove parallel modifier from e2e tests ([#15172](https://github.com/GoogleChrome/lighthouse/pull/15172))
+* issues-mixed-content: use new site for testing ([#15241](https://github.com/GoogleChrome/lighthouse/pull/15241))
+* runner: drop usages of legacy runner ([#15047](https://github.com/GoogleChrome/lighthouse/pull/15047))
+* smoke: expect server-response-time to be greater than 0 ([#15188](https://github.com/GoogleChrome/lighthouse/pull/15188))
+
 <a name="10.3.0"></a>
 # 10.3.0 (2023-06-13)
 [Full Changelog](https://github.com/GoogleChrome/lighthouse/compare/v10.2.0...v10.3.0)

cli/test/fixtures/a11y/a11y_tester.html

@@ -24,6 +24,14 @@
         aria-checked="true">
       </div>
     </section>
+    <p>aria-command-name</p>
+    <section>
+      <div id="aria-command-name" role="button"></div>
+    </section>
+    <p>aria-dialog-name</p>
+    <section>
+      <div role="alertdialog" id="aria-dialog-name" aria-label=""></div>
+    </section>
     <p>aria-hidden-focus</p>
     <section>
       <div id="aria-hidden-focus" aria-hidden="true">
@@ -44,10 +52,6 @@
         <div id="aria-treeitem-name" role="treeitem"></div>
       </div>
     </section>
-    <p>aria-command-name</p>
-    <section>
-      <div id="aria-command-name" role="button"></div>
-    </section>
     <p>aria-tooltip-name</p>
     <section>
       <div id="aria-tooltip-name" role="tooltip"></div>
@@ -84,6 +88,12 @@
     <section>
       <div role="foo"></div>
     </section>
+    <p>aria-text</p>
+    <section>
+      <span role="text" id="aria-text">
+        <a href="google.com">Not announced</a>
+      </span>
+    </section>
     <p>aria-valid-attr</p>
     <section>
       <div
@@ -135,6 +145,10 @@
       <div id="duplicate-id-aria"></div>
       <input type="text" aria-labelledby="duplicate-id-aria"/>
     </section>
+    <p>empty-heading</p>
+    <section>
+      <h1 id="empty-heading"></h1>
+    </section>
     <p>form-field-multiple-labels</p>
     <section>
       <label for="form-field-multiple-labels" id="label1">label1</label>
@@ -150,6 +164,11 @@
       <h1>top header</h1>
       <h3>sub-sub-header</h3>
     </section>
+    <p>identical-links-same-purpose</p>
+    <section>
+      <a id="identical-links-same-purpose-1" href="https://example.com/" aria-label="a link"></a>
+      <a id="identical-links-same-purpose-2" href="https://google.com/" aria-label="a link"></a>
+    </section>
     <p>image-alt</p>
     <section>
       <img
@@ -180,6 +199,19 @@ <h3>sub-sub-header</h3>
       <table id="layout-table">
       </table>
     </section>
+    <p>link-in-text-block</p>
+    <section>
+      <p>
+          paragraph text
+          <a
+            style="text-decoration: none; color: blue"
+            href="#"
+            id="link-in-text-block">
+            link text
+          </a>
+          paragraph text
+      </p>
+    </section>
     <p>link-name</p>
     <section>
       <a id="link-name" href="google.com"></a>
@@ -198,6 +230,10 @@ <h3>sub-sub-header</h3>
     <section>
       <object id="object-alt" data="data:text/html,data"></object>
     </section>
+    <p>select-name</p>
+    <section>
+      <select id="select-name"></select>
+    </section>
     <p>tabindex</p>
     <section>
       <div
@@ -212,6 +248,11 @@ <h3>sub-sub-header</h3>
         <tr><td>foo</td><td>foo</td><tr>
       </table>
     </section>
+    <p>target-size</p>
+    <section>
+      <button id="target-size-1">+</button>
+      <span role="button" tabindex="0" id="target-size-2">o</span>
+    </section>
     <p>td-has-header</p>
     <section>
       <table id="td-has-header">

cli/test/smokehouse/frontends/smokehouse-bin.js

@@ -25,6 +25,8 @@ import {runSmokehouse, getShardedDefinitions} from '../smokehouse.js';
 import {updateTestDefnFormat} from './back-compat-util.js';
 import {LH_ROOT} from '../../../../root.js';
 import exclusions from '../config/exclusions.js';
+import {saveArtifacts} from '../../../../core/lib/asset-saver.js';
+import {saveLhr} from '../../../../core/lib/asset-saver.js';
 
 const coreTestDefnsPath =
   path.join(LH_ROOT, 'cli/test/smokehouse/core-tests.js');
@@ -230,19 +232,23 @@ async function begin() {
   if (!smokehouseResult.success) {
     const failedTestResults = smokehouseResult.testResults.filter(r => r.failed);
 
-    // For CI, save failed runs to directory to be uploaded.
-    if (process.env.CI) {
-      const failuresDir = `${LH_ROOT}/.tmp/smokehouse-ci-failures`;
-      fs.mkdirSync(failuresDir, {recursive: true});
-
-      for (const testResult of failedTestResults) {
-        for (let i = 0; i < testResult.runs.length; i++) {
-          const run = testResult.runs[i];
-          fs.writeFileSync(`${failuresDir}/${testResult.id}-${i}.json`, JSON.stringify({
-            ...run,
-            lighthouseLog: run.lighthouseLog.split('\n'),
-            assertionLog: run.assertionLog.split('\n'),
-          }, null, 2));
+    // Save failed runs to directory. In CI, this is uploaded as an artifact.
+    const failuresDir = `${LH_ROOT}/.tmp/smokehouse-failures`;
+    fs.rmSync(failuresDir, {recursive: true, force: true});
+    fs.mkdirSync(failuresDir);
+
+    for (const testResult of failedTestResults) {
+      for (let i = 0; i < testResult.runs.length; i++) {
+        const runDir = `${failuresDir}/${i}/${testResult.id}`;
+        fs.mkdirSync(runDir, {recursive: true});
+
+        const run = testResult.runs[i];
+        await saveArtifacts(run.artifacts, runDir);
+        await saveLhr(run.lhr, runDir);
+        fs.writeFileSync(`${runDir}/assertionLog.txt`, run.assertionLog);
+        fs.writeFileSync(`${runDir}/lighthouseLog.txt`, run.lighthouseLog);
+        if (run.networkRequests) {
+          fs.writeFileSync(`${runDir}/networkRequests.txt`, run.networkRequests.join('\n'));
         }
       }
     }

cli/test/smokehouse/lighthouse-runners/bundle.js

@@ -82,7 +82,7 @@ async function runBundledLighthouse(url, config, testRunnerOptions) {
 
   // Run Lighthouse.
   try {
-    const logLevel = testRunnerOptions.isDebug ? 'info' : undefined;
+    const logLevel = testRunnerOptions.isDebug ? 'verbose' : 'info';
     let runnerResult;
     if (testRunnerOptions.useLegacyNavigation) {
       const connection = new CriConnection(port);
@@ -124,12 +124,10 @@ async function runLighthouse(url, config, testRunnerOptions = {}) {
   worker.stdout.setEncoding('utf8');
   worker.stderr.setEncoding('utf8');
   worker.stdout.addListener('data', (data) => {
-    process.stdout.write(data);
-    logs.push(`STDOUT: ${data}`);
+    logs.push(`[STDOUT] ${data}`);
   });
   worker.stderr.addListener('data', (data) => {
-    process.stderr.write(data);
-    logs.push(`STDERR: ${data}`);
+    logs.push(`[STDERR] ${data}`);
   });
   const [workerResponse] = await once(worker, 'message');
   const log = logs.join('') + '\n';