Merge pull request #38 from storybookjs/test-hooks

Add setPreRender/setPostRender hooks & example
storybookjs · Feb 3, 2022 · 52c0571 · 52c0571
2 parents 86ed1e9 + ad367be
commit 52c0571
Show file tree

Hide file tree

Showing 22 changed files with 361 additions and 45 deletions.
diff --git a/README.md b/README.md
@@ -8,12 +8,15 @@ Storybook test runner turns all of your stories into executable tests.
   - [Table of Contents](#table-of-contents)
   - [Features](#features)
   - [Getting started](#getting-started)
+  - [CLI Options](#cli-options)
   - [Configuration](#configuration)
   - [Running against a deployed Storybook](#running-against-a-deployed-storybook)
     - [Stories.json mode](#storiesjson-mode)
   - [Running in CI](#running-in-ci)
     - [1. Running against deployed Storybooks on Github Actions deployment](#1-running-against-deployed-storybooks-on-github-actions-deployment)
     - [2. Running against locally built Storybooks in CI](#2-running-against-locally-built-storybooks-in-ci)
+  - [Experimental test hook API](#experimental-test-hook-api)
+    - [Image snapshot recipe](#image-snapshot-recipe)
   - [Troubleshooting](#troubleshooting)
     - [The test runner seems flaky and keeps timing out](#the-test-runner-seems-flaky-and-keeps-timing-out)
     - [Adding the test runner to other CI environments](#adding-the-test-runner-to-other-ci-environments)
@@ -98,16 +101,16 @@ yarn test-storybook
 Usage: test-storybook [options]
 ```
 
-| Options                         | Description                                                                                                                                                                 |
-| ------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `--help`                        | Output usage information <br/>`test-storybook --help`                                                                                                                      |
-| `-s`, `--stories-json` | Run in stories json mode (requires a compatible Storybook) <br/>`test-storybook --stories-json`                                                                                  |
-| `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from <br/>`test-storybook -c .storybook`                                                                                  |
-| `--watch` | Run in watch mode <br/>`test-storybook --watch`                                                                                  |
-| `--maxWorkers [amount]` | Specifies the maximum number of workers the worker-pool will spawn for running tests <br/>`test-storybook --maxWorkers=2`                                                                                  |
-| `--no-cache` | Disable the cache <br/>`test-storybook --no-cache`                                                                                  |
-| `--clearCache` | Deletes the Jest cache directory and then exits without running tests <br/>`test-storybook --clearCache`                                                                                  |
-| `--verbose` | Display individual test results with the test suite hierarchy <br/>`test-storybook --verbose`                                                                                  |
+| Options                         | Description                                                                                                               |
+| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
+| `--help`                        | Output usage information <br/>`test-storybook --help`                                                                     |
+| `-s`, `--stories-json`          | Run in stories json mode (requires a compatible Storybook) <br/>`test-storybook --stories-json`                           |
+| `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from <br/>`test-storybook -c .storybook`                                 |
+| `--watch`                       | Run in watch mode <br/>`test-storybook --watch`                                                                           |
+| `--maxWorkers [amount]`         | Specifies the maximum number of workers the worker-pool will spawn for running tests <br/>`test-storybook --maxWorkers=2` |
+| `--no-cache`                    | Disable the cache <br/>`test-storybook --no-cache`                                                                        |
+| `--clearCache`                  | Deletes the Jest cache directory and then exits without running tests <br/>`test-storybook --clearCache`                  |
+| `--verbose`                     | Display individual test results with the test suite hierarchy <br/>`test-storybook --verbose`                             |
 
 ## Configuration
 
@@ -245,6 +248,59 @@ jobs:
 
 > **_NOTE:_** Building Storybook locally makes it simple to test Storybooks that could be available remotely, but are under authentication layers. If you also deploy your Storybooks somewhere (e.g. Chromatic, Vercel, etc.), the Storybook URL can still be useful with the test-runner. You can pass it to the `REFERENCE_URL` environment variable when running the test-storybook command, and if a story fails, the test-runner will provide a helpful message with the link to the story in your published Storybook instead.
 
+## Experimental test hook API
+
+The test runner renders a story and executes its [play function](https://storybook.js.org/docs/react/writing-stories/play-function) if one exists. However, there are certain behaviors that are not possible to achieve via the play function, which executes in the browser. For example, if you want the test runner to take visual snapshots for you, this is something that is possible via Playwright/Jest, but must be executed in Node.
+
+To enable use cases like visual or DOM snapshots, the test runner exports test hooks that can be overridden globally. These hooks give you access to the test lifecycle before and after the story is rendered.
+
+The hooks, `preRender` and `postRender`, are functions that take a [Playwright Page](https://playwright.dev/docs/pages) and a context object with the current story `id`, `title`, and `name`. They are globally settable by `@storybook/test-runner`'s `setPreRender` and `setPostRender` APIs.
+
+> **NOTE:** These test hooks are experimental and may be subject to breaking changes. We encourage you to test as much as possible within the story's play function.
+
+To visualize the test lifecycle, consider a simplified version of the test code automatically generated for each story in your Storybook:
+
+```js
+it('button--basic', async () => {
+  // filled in with data for the current story
+  const context = { id: 'button--basic', title: 'Button', name: 'Basic' };
+
+  // playwright page https://playwright.dev/docs/pages
+  await page.goto(STORYBOOK_URL);
+
+  // pre-render hook
+  if (preRender) await preRender(page, context);
+
+  // render the story and run its play function (if applicable)
+  await page.execute('render', context);
+
+  // post-render hook
+  if (postRender) await postRender(page, context);
+});
+```
+
+### Image snapshot recipe
+
+If you want to make the test runner take image snapshots, the following recipe uses test hooks in `jest-setup.js` to do it:
+
+```js
+const { toMatchImageSnapshot } = require('jest-image-snapshot');
+const { setPostRender } = require('@storybook/test-runner');
+
+expect.extend({ toMatchImageSnapshot });
+
+// use custom directory/id to align CSF and stories.json mode outputs
+const customSnapshotsDir = `${process.cwd()}/__snapshots__`;
+
+setPostRender(async (page, context) => {
+  const image = await page.screenshot();
+  expect(image).toMatchImageSnapshot({
+    customSnapshotsDir,
+    customSnapshotIdentifier: context.id,
+  });
+});
+```
+
 ## Troubleshooting
 
 #### The test runner seems flaky and keeps timing out

diff --git a/__snapshots__/basic-button--demo-snap.png b/__snapshots__/basic-button--demo-snap.png
diff --git a/__snapshots__/basic-button--find-by-snap.png b/__snapshots__/basic-button--find-by-snap.png
diff --git a/__snapshots__/basic-button--primary-snap.png b/__snapshots__/basic-button--primary-snap.png
diff --git a/__snapshots__/basic-button--wait-for-element-to-be-removed-snap.png b/__snapshots__/basic-button--wait-for-element-to-be-removed-snap.png
diff --git a/__snapshots__/basic-button--wait-for-snap.png b/__snapshots__/basic-button--wait-for-snap.png
diff --git a/__snapshots__/basic-button--with-loaders-snap.png b/__snapshots__/basic-button--with-loaders-snap.png
diff --git a/__snapshots__/example-header--logged-in-snap.png b/__snapshots__/example-header--logged-in-snap.png
diff --git a/__snapshots__/example-header--logged-out-snap.png b/__snapshots__/example-header--logged-out-snap.png
diff --git a/__snapshots__/example-page--logged-in-snap.png b/__snapshots__/example-page--logged-in-snap.png
diff --git a/__snapshots__/example-page--logged-out-snap.png b/__snapshots__/example-page--logged-out-snap.png
diff --git a/jest-setup.js b/jest-setup.js
@@ -0,0 +1,17 @@
+const { toMatchImageSnapshot } = require('jest-image-snapshot');
+const { setPostRender } = require('./dist/cjs');
+
+expect.extend({ toMatchImageSnapshot });
+
+// use custom directory/id to align CSF and stories.json mode outputs
+const customSnapshotsDir = `${process.cwd()}/__snapshots__`;
+
+setPostRender(async (page, context) => {
+  const image = await page.screenshot();
+  expect(image).toMatchImageSnapshot({
+    customSnapshotsDir,
+    customSnapshotIdentifier: context.id,
+    failureThreshold: 0.03,
+    failureThresholdType: 'percent',
+  });
+});
diff --git a/package.json b/package.json
@@ -83,6 +83,7 @@
     "dedent": "^0.7.0",
     "jest": "^27.0.6",
     "jest-environment-jsdom": "^27.0.6",
+    "jest-image-snapshot": "^4.5.1",
     "prettier": "^2.3.1",
     "prop-types": "^15.7.2",
     "react": "^17.0.1",
@@ -107,6 +108,7 @@
     "@storybook/csf": "0.0.2--canary.87bc651.0",
     "@storybook/csf-tools": "^6.4.14",
     "commander": "^9.0.0",
+    "global": "^4.4.0",
     "jest-playwright-preset": "^1.7.0",
     "node-fetch": "^2",
     "playwright": "^1.14.0",

diff --git a/src/csf/transformCsf.ts b/src/csf/transformCsf.ts
@@ -13,8 +13,9 @@ export interface TestContext {
   title: t.Literal;
   id: t.Literal;
 }
-type FilePrefixer = () => t.Statement[];
-type TestPrefixer = (context: TestContext) => t.Statement[];
+type TemplateResult = t.Statement | t.Statement[];
+type FilePrefixer = () => TemplateResult;
+type TestPrefixer = (context: TestContext) => TemplateResult;
 
 interface TransformOptions {
   clearBody?: boolean;
@@ -38,8 +39,7 @@ const prefixFunction = (
     id: t.stringLiteral(toId(title, name)),
   };
 
-  // instead, let's just make the prefixer return the function
-  const result = testPrefixer(context);
+  const result = makeArray(testPrefixer(context));
   const stmt = result[1] as t.ExpressionStatement;
   return stmt.expression;
 };
@@ -69,6 +69,9 @@ const makeDescribe = (key: string, tests: t.Statement[]): t.Statement | null =>
   );
 };
 
+const makeArray = (templateResult: TemplateResult) =>
+  Array.isArray(templateResult) ? templateResult : [templateResult];
+
 export const transformCsf = (
   code: string,
   {
@@ -110,7 +113,7 @@ export const transformCsf = (
 
   // FIXME: insert between imports
   if (filePrefixer) {
-    const { code: prefixCode } = generate(t.program(filePrefixer()), {});
+    const { code: prefixCode } = generate(t.program(makeArray(filePrefixer())), {});
     result = `${prefixCode}\n`;
   }
   if (!clearBody) result = `${result}${code}\n`;

diff --git a/src/index.ts b/src/index.ts
@@ -0,0 +1 @@
+export * from './playwright/hooks';
diff --git a/src/playwright/hooks.ts b/src/playwright/hooks.ts
@@ -0,0 +1,18 @@
+import global from 'global';
+import type { Page } from 'playwright';
+
+export type TestContext = {
+  id: string;
+  title: string;
+  name: string;
+};
+
+export type TestHook = (page: Page, context: TestContext) => Promise<void>;
+
+export const setPreRender = (preRender: TestHook) => {
+  global.__sbPreRender = preRender;
+};
+
+export const setPostRender = (postRender: TestHook) => {
+  global.__sbPostRender = postRender;
+};
diff --git a/src/playwright/transformPlaywright.test.ts b/src/playwright/transformPlaywright.test.ts
@@ -48,10 +48,17 @@ describe('Playwright', () => {
         filename
       )
     ).toMatchInlineSnapshot(`
+      import global from 'global';
+
       if (!require.main) {
         describe("foo/bar", () => {
         describe("A", () => {
           it("play-test", async () => {
+            const context = {
+              id: "foo-bar--a",
+              title: "foo/bar",
+              name: "A"
+            };
             page.on('pageerror', err => {
               page.evaluate(({
                 id,
@@ -61,11 +68,23 @@ describe('Playwright', () => {
                 err: err.message
               });
             });
-            return page.evaluate(({
-              id
-            }) => __test(id), {
+
+            if (global.__sbPreRender) {
+              await global.__sbPreRender(page, context);
+            }
+
+            const result = await page.evaluate(({
+              id,
+              hasPlayFn
+            }) => __test(id, hasPlayFn), {
               id: "foo-bar--a"
             });
+
+            if (global.__sbPostRender) {
+              await global.__sbPostRender(page, context);
+            }
+
+            return result;
           });
         });
       });
@@ -82,10 +101,17 @@ describe('Playwright', () => {
         filename
       )
     ).toMatchInlineSnapshot(`
+      import global from 'global';
+
       if (!require.main) {
         describe("foo/bar", () => {
         describe("A", () => {
           it("smoke-test", async () => {
+            const context = {
+              id: "foo-bar--a",
+              title: "foo/bar",
+              name: "A"
+            };
             page.on('pageerror', err => {
               page.evaluate(({
                 id,
@@ -95,11 +121,23 @@ describe('Playwright', () => {
                 err: err.message
               });
             });
-            return page.evaluate(({
-              id
-            }) => __test(id), {
+
+            if (global.__sbPreRender) {
+              await global.__sbPreRender(page, context);
+            }
+
+            const result = await page.evaluate(({
+              id,
+              hasPlayFn
+            }) => __test(id, hasPlayFn), {
               id: "foo-bar--a"
             });
+
+            if (global.__sbPostRender) {
+              await global.__sbPostRender(page, context);
+            }
+
+            return result;
           });
         });
       });
@@ -117,10 +155,17 @@ describe('Playwright', () => {
         filename
       )
     ).toMatchInlineSnapshot(`
+      import global from 'global';
+
       if (!require.main) {
         describe("Example/Header", () => {
         describe("A", () => {
           it("smoke-test", async () => {
+            const context = {
+              id: "example-header--a",
+              title: "Example/Header",
+              name: "A"
+            };
             page.on('pageerror', err => {
               page.evaluate(({
                 id,
@@ -130,11 +175,23 @@ describe('Playwright', () => {
                 err: err.message
               });
             });
-            return page.evaluate(({
-              id
-            }) => __test(id), {
+
+            if (global.__sbPreRender) {
+              await global.__sbPreRender(page, context);
+            }
+
+            const result = await page.evaluate(({
+              id,
+              hasPlayFn
+            }) => __test(id, hasPlayFn), {
               id: "example-header--a"
             });
+
+            if (global.__sbPostRender) {
+              await global.__sbPostRender(page, context);
+            }
+
+            return result;
           });
         });
       });

diff --git a/src/playwright/transformPlaywright.ts b/src/playwright/transformPlaywright.ts
@@ -6,17 +6,33 @@ import { autoTitle } from '@storybook/store';
 import { getStorybookMain } from '../util/cli';
 import { transformCsf } from '../csf/transformCsf';
 
+const filePrefixer = template(`
+  import global from 'global';
+`);
+
 export const testPrefixer = template(
   `
     console.log({ id: %%id%%, title: %%title%%, name: %%name%%, storyExport: %%storyExport%% });
     async () => {
+      const context = { id: %%id%%, title: %%title%%, name: %%name%% };
+
       page.on('pageerror', (err) => {
         page.evaluate(({ id, err }) => __throwError(id, err), { id: %%id%%, err: err.message });
       });
 
-      return page.evaluate(({ id }) => __test(id), {
-        id: %%id%%
+      if(global.__sbPreRender) {
+        await global.__sbPreRender(page, context);
+      }
+
+      const result = await page.evaluate(({ id, hasPlayFn }) => __test(id, hasPlayFn), {
+        id: %%id%%,
       });
+
+      if(global.__sbPostRender) {
+        await global.__sbPostRender(page, context);
+      }
+
+      return result;
     }
   `,
   {
@@ -46,6 +62,7 @@ const getDefaultTitle = (filename: string) => {
 export const transformPlaywright = (src: string, filename: string) => {
   const defaultTitle = getDefaultTitle(filename);
   const result = transformCsf(src, {
+    filePrefixer,
     // @ts-ignore
     testPrefixer,
     insertTestIfEmpty: true,