diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1956f2f3..518c023a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3,10 +3,15 @@ name: Nightly checks # runs every day at midnight on: schedule: - - cron: "0 0 * * *" + - cron: '0 0 * * *' + workflow_dispatch: + # To test fixes on push rather than wait for the scheduling + push: + branches: + - fix/nightly jobs: - test_storybook_prerelease: + assert_test_runner: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -24,15 +29,14 @@ jobs: echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV - - name: Upgrade to storybook@next + - name: Upgrade to storybook@future run: | - npx storybook upgrade --prerelease + npx storybook@future upgrade --prerelease --yes # TODO: This should not be necessary once @storybook/csf is properly updated - name: Fix local @storybook/csf version run: | yarn add @storybook/csf@0.0.2--canary.4566f4d.1 - - name: Run test runner uses: mathiasvr/command-output@v1 id: tests @@ -52,7 +56,7 @@ jobs: id: slack uses: slackapi/slack-github-action@v1.19.0 with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID }}" + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' payload: | { "blocks": [ @@ -92,3 +96,101 @@ jobs: } env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + assert_test_runner_failures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Use Node.js 14.x + uses: actions/setup-node@v1 + with: + node-version: 14.x + + - name: Install dependencies + uses: bahmutov/npm-install@v1 + + - name: Get current version of Storybook + run: | + echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Upgrade to storybook@future + run: | + npx storybook@future upgrade --prerelease --yes + + # TODO: This should not be necessary once @storybook/csf is properly updated + - name: Fix local @storybook/csf version + run: | + yarn add @storybook/csf@0.0.2--canary.4566f4d.1 + + - name: Run test runner and expect failure + uses: mathiasvr/command-output@v1 + with: + run: | + yarn build + yarn test-storybook:ci-failures + + - name: Process test results + if: ${{ always() }} + id: tests + uses: sergeysova/jq-action@v2 + with: + cmd: 'jq .numPassedTests test-results.json -r' + + - name: Set failure check to env + if: ${{ always() }} + run: | + echo "FAILED=${{ steps.tests.outputs.value > 0 }}" >> $GITHUB_ENV + + - name: Get prerelease version of Storybook + if: ${{ always() && env.FAILED == 'true' }} + run: | + echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV + echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV + + - name: Report if any test passes + if: ${{ always() && env.FAILED == 'true' }} + id: slack + uses: slackapi/slack-github-action@v1.19.0 + with: + channel-id: '${{ secrets.SLACK_CHANNEL_ID }}' + payload: | + { + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:", + "emoji": true + } + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}" + }, + { + "type": "mrkdwn", + "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}" + } + ], + "accessory": { + "type": "button", + "text": { + "type": "plain_text", + "text": "View failure", + "emoji": true + }, + "value": "view_failure", + "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + "action_id": "button-action" + } + } + ] + } + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d84250a..1eccc71d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -25,4 +25,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} NPM_TOKEN: ${{ secrets.NPM_TOKEN }} run: | - yarn release \ No newline at end of file + yarn release diff --git a/.github/workflows/stress-test.yml b/.github/workflows/stress-test.yml index 63424647..1fa805e9 100644 --- a/.github/workflows/stress-test.yml +++ b/.github/workflows/stress-test.yml @@ -2,7 +2,7 @@ name: Stress Tests on: pull_request: - types: [ labeled ] + types: [labeled] jobs: test: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3f7ed98f..2b3e15ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,7 @@ jobs: run: | yarn build yarn test-storybook:ci-coverage - + - name: Generate code coverage uses: codecov/codecov-action@v2 with: diff --git a/.gitignore b/.gitignore index 46710295..86ff7083 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ build-storybook.log stories/atoms/StressTest.stories.js yarn-error.log .nyc_output -coverage \ No newline at end of file +coverage +test-results.json \ No newline at end of file diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 00000000..d24fdfc6 --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,4 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +npx lint-staged diff --git a/.prettierignore b/.prettierignore index 1521c8b7..8db646af 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,8 @@ dist +node_modules +storybook-static +.cache +.env +*.snap +__snapshots__ +CHANGELOG.md \ No newline at end of file diff --git a/.storybook/main.js b/.storybook/main.js index e472b171..73588dd5 100644 --- a/.storybook/main.js +++ b/.storybook/main.js @@ -1,4 +1,4 @@ -const stories = [ +let stories = [ '../stories/docs/**/*.stories.mdx', // default title prefix { @@ -18,14 +18,18 @@ if (process.env.STRESS_TEST) { stories.push('../stories/stress-test/*.stories.@(js|jsx|ts|tsx)'); } +if (process.env.TEST_FAILURES) { + stories = ['../stories/expected-failures/*.stories.@(js|jsx|ts|tsx)']; +} + const addons = [ process.env.WITHOUT_DOCS ? { - name: '@storybook/addon-essentials', - options: { - docs: false, - }, - } + name: '@storybook/addon-essentials', + options: { + docs: false, + }, + } : '@storybook/addon-essentials', '@storybook/addon-interactions', '@storybook/addon-coverage', @@ -39,6 +43,6 @@ module.exports = { buildStoriesJson: true, }, core: { - disableTelemetry: true - } + disableTelemetry: true, + }, }; diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts index 76d89258..c36b5b0d 100644 --- a/.storybook/test-runner.ts +++ b/.storybook/test-runner.ts @@ -4,6 +4,7 @@ import type { TestRunnerConfig } from '../dist/ts'; const snapshotsDir = process.env.SNAPSHOTS_DIR || '__snapshots__'; const customSnapshotsDir = `${process.cwd()}/${snapshotsDir}`; +const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true'; const config: TestRunnerConfig = { setup() { @@ -17,6 +18,10 @@ const config: TestRunnerConfig = { return; } + if (skipSnapshots) { + return; + } + // Visual snapshot tests const image = await page.screenshot({ fullPage: true }); expect(image).toMatchImageSnapshot({ diff --git a/.vscode/settings.json b/.vscode/settings.json index 20ecf671..623432bf 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,4 +5,4 @@ "titleBar.activeBackground": "#004752", "titleBar.activeForeground": "#ECFCFF" } -} \ No newline at end of file +} diff --git a/README.md b/README.md index ca8e83b8..cb2726df 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ See the announcement of Interaction Testing with Storybook in detail in [this bl The Storybook test runner uses Jest as a runner, and Playwright as a testing framework. Each one of your `.stories` files is transformed into a spec file, and each story becomes a test, which is run in a headless browser. The test runner is simple in design – it just visits each story from a running Storybook instance and makes sure the component is not failing: + - For stories without a `play` function, it verifies whether the story rendered without any errors. This is essentially a smoke test. - For those with a `play` function, it also checks for errors in the `play` function and that all assertions passed. This is essentially an [interaction test](https://storybook.js.org/docs/react/writing-tests/interaction-testing#write-an-interaction-test). @@ -127,23 +128,25 @@ yarn test-storybook Usage: test-storybook [options] ``` -| Options | Description | -| ------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -| `--help` | Output usage information
`test-storybook --help` | -| `-i`, `--index-json` | Run in index json mode. Automatically detected (requires a compatible Storybook)
`test-storybook --index-json` | -| `--no-index-json` | Disables index json mode
`test-storybook --no-index-json` | -| `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from
`test-storybook -c .storybook` | -| `--watch` | Watch files for changes and rerun tests related to changed files.
`test-storybook --watch` | -| `--watchAll` | Watch files for changes and rerun all tests when something changes.
`test-storybook --watchAll` | -| `--coverage` | Indicates that test coverage information should be collected and reported in the output
`test-storybook --coverage` | -| `--url` | Define the URL to run tests in. Useful for custom Storybook URLs
`test-storybook --url http://the-storybook-url-here.com` | -| `--browsers` | Define browsers to run tests in. One or multiple of: chromium, firefox, webkit
`test-storybook --browsers firefox chromium` | -| `--maxWorkers [amount]` | Specifies the maximum number of workers the worker-pool will spawn for running tests
`test-storybook --maxWorkers=2` | -| `--no-cache` | Disable the cache
`test-storybook --no-cache` | -| `--clearCache` | Deletes the Jest cache directory and then exits without running tests
`test-storybook --clearCache` | -| `--verbose` | Display individual test results with the test suite hierarchy
`test-storybook --verbose` | -| `-u`, `--updateSnapshot` | Use this flag to re-record every snapshot that fails during this test run
`test-storybook -u` | -| `--eject` | Creates a local configuration file to override defaults of the test-runner
`test-storybook --eject` | +| Options | Description | +| ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | +| `--help` | Output usage information
`test-storybook --help` | +| `-i`, `--index-json` | Run in index json mode. Automatically detected (requires a compatible Storybook)
`test-storybook --index-json` | +| `--no-index-json` | Disables index json mode
`test-storybook --no-index-json` | +| `-c`, `--config-dir [dir-name]` | Directory where to load Storybook configurations from
`test-storybook -c .storybook` | +| `--watch` | Watch files for changes and rerun tests related to changed files.
`test-storybook --watch` | +| `--watchAll` | Watch files for changes and rerun all tests when something changes.
`test-storybook --watchAll` | +| `--coverage` | Indicates that test coverage information should be collected and reported in the output
`test-storybook --coverage` | +| `--url` | Define the URL to run tests in. Useful for custom Storybook URLs
`test-storybook --url http://the-storybook-url-here.com` | +| `--browsers` | Define browsers to run tests in. One or multiple of: chromium, firefox, webkit
`test-storybook --browsers firefox chromium` | +| `--maxWorkers [amount]` | Specifies the maximum number of workers the worker-pool will spawn for running tests
`test-storybook --maxWorkers=2` | +| `--no-cache` | Disable the cache
`test-storybook --no-cache` | +| `--clearCache` | Deletes the Jest cache directory and then exits without running tests
`test-storybook --clearCache` | +| `--verbose` | Display individual test results with the test suite hierarchy
`test-storybook --verbose` | +| `-u`, `--updateSnapshot` | Use this flag to re-record every snapshot that fails during this test run
`test-storybook -u` | +| `--eject` | Creates a local configuration file to override defaults of the test-runner
`test-storybook --eject` | +| `--json` | Prints the test results in JSON. This mode will send all other test output and user messages to stderr.
`test-storybook --json` | +| `--outputFile` | Write test results to a file when the --json option is also specified.
`test-storybook --json --outputFile results.json` | ## Configuration @@ -297,11 +300,11 @@ The test runner supports code coverage with the `--coverage` flag or `STORYBOOK_ ### 1 - Instrument the code -Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. This is done by configuring [istanbul](https://istanbul.js.org/) in your Storybook. You can achieve that in two different ways: +Given that your components' code runs in the context of a real browser, they have to be instrumented so that the test runner is able to collect coverage. This is done by configuring [istanbul](https://istanbul.js.org/) in your Storybook. You can achieve that in two different ways: #### Using @storybook/addon-coverage -For select frameworks (React, Preact, HTML, Web components and Vue) you can use the [@storybook/addon-coverage](https://github.com/storybookjs/addon-coverage) addon, which will automatically configure the plugin for you. +For select frameworks with Webpack (React, Preact, HTML, Web components and Vue) you can use the [@storybook/addon-coverage](https://github.com/storybookjs/addon-coverage) addon, which will automatically configure the plugin for you. Install `@storybook/addon-coverage`: @@ -315,9 +318,7 @@ And register it in your `.storybook/main.js` file: // .storybook/main.js module.exports = { // ...rest of your code here - addons: [ - "@storybook/addon-coverage", - ] + addons: ['@storybook/addon-coverage'], }; ``` @@ -325,7 +326,7 @@ The addon has default options that might suffice to your project, however if you #### Manually configuring istanbul -Some frameworks or Storybook builders might not automatically accept babel plugins. In that case, you will have to manually configure whatever flavor of [istanbul](https://istanbul.js.org/) (rollup, vite, webpack loader) your project might require. +Some frameworks or Storybook builders (e.g. Vite) might not automatically accept babel plugins. In that case, you will have to manually configure whatever flavor of [istanbul](https://istanbul.js.org/) (Rollup, Vite, Webpack loader) your project might require. You can find recipes in [this repository](https://github.com/yannbf/storybook-coverage-recipes) that include many different configurations and steps on how to set up coverage in each of them. ### 2 - Run tests with --coverage flag @@ -357,7 +358,7 @@ If you want certain parts of your code to be deliberately ignored, you can use i ### 3 - Merging code coverage with coverage from other tools -The test runner reports coverage related to the `coverage/storybook/coverage-storybook.json` file. This is by design, showing you the coverage which is tested while running Storybook. +The test runner reports coverage related to the `coverage/storybook/coverage-storybook.json` file. This is by design, showing you the coverage which is tested while running Storybook. Now, you might have other tests (e.g. unit tests) which are _not_ covered in Storybook but are covered when running tests with Jest, which you might also generate coverage files from, for instance. In such cases, if you are using tools like [Codecov](https://codecov.io/) to automate reporting, the coverage files will be detected automatically and if there are multiple files in the coverage folder, they will be merged automatically. @@ -485,29 +486,29 @@ You can use it for multiple use cases, and here's an example that combines the s // .storybook/test-runner.js const { getStoryContext } = require('@storybook/test-runner'); const { injectAxe, checkA11y } = require('axe-playwright'); - + module.exports = { - async preRender(page, context) { - await injectAxe(page); - }, - async postRender(page, context) { - // Get entire context of a story, including parameters, args, argTypes, etc. - const storyContext = await getStoryContext(page, context); - - // Do not test a11y for stories that disable a11y - if (storyContext.parameters?.a11y?.disable) { - return; - } - - await checkA11y(page, '#root', { - detailedReport: true, - detailedReportOptions: { - html: true, - }, - // pass axe options defined in @storybook/addon-a11y - axeOptions: storyContext.parameters?.a11y?.options - }) - }, + async preRender(page, context) { + await injectAxe(page); + }, + async postRender(page, context) { + // Get entire context of a story, including parameters, args, argTypes, etc. + const storyContext = await getStoryContext(page, context); + + // Do not test a11y for stories that disable a11y + if (storyContext.parameters?.a11y?.disable) { + return; + } + + await checkA11y(page, '#root', { + detailedReport: true, + detailedReportOptions: { + html: true, + }, + // pass axe options defined in @storybook/addon-a11y + axeOptions: storyContext.parameters?.a11y?.options, + }); + }, }; ``` diff --git a/__snapshots__/pages-page--logged-in-snap.png b/__snapshots__/pages-page--logged-in-snap.png index 55c368e8..ac28c9d9 100644 Binary files a/__snapshots__/pages-page--logged-in-snap.png and b/__snapshots__/pages-page--logged-in-snap.png differ diff --git a/__snapshots__/pages-page--logged-out-snap.png b/__snapshots__/pages-page--logged-out-snap.png index 7e434076..699c4295 100644 Binary files a/__snapshots__/pages-page--logged-out-snap.png and b/__snapshots__/pages-page--logged-out-snap.png differ diff --git a/bin/test-storybook.js b/bin/test-storybook.js index 44e67407..e70acf32 100755 --- a/bin/test-storybook.js +++ b/bin/test-storybook.js @@ -40,7 +40,7 @@ const cleanup = () => { let isWatchMode = false; async function reportCoverage() { if (isWatchMode || process.env.STORYBOOK_COLLECT_COVERAGE !== 'true') { - return + return; } const coverageFolderE2E = path.resolve(process.cwd(), '.nyc_output'); @@ -48,7 +48,7 @@ async function reportCoverage() { // in case something goes wrong and .nyc_output does not exist, bail if (!fs.existsSync(coverageFolderE2E)) { - return + return; } // if there's no coverage folder, create one @@ -57,22 +57,21 @@ async function reportCoverage() { } // move the coverage files from .nyc_output folder (coming from jest-playwright) to coverage, then delete .nyc_output - fs.renameSync( - `${coverageFolderE2E}/coverage.json`, - `${coverageFolder}/coverage-storybook.json`, - ); + fs.renameSync(`${coverageFolderE2E}/coverage.json`, `${coverageFolder}/coverage-storybook.json`); fs.rmSync(coverageFolderE2E, { recursive: true }); // --skip-full in case we only want to show not fully covered code // --check-coverage if we want to break if coverage reaches certain threshold // .nycrc will be respected for thresholds etc. https://www.npmjs.com/package/nyc#coverage-thresholds - execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, { stdio: 'inherit' }) + execSync(`npx nyc report --reporter=text -t ${coverageFolder} --report-dir ${coverageFolder}`, { + stdio: 'inherit', + }); } const onProcessEnd = () => { cleanup(); reportCoverage(); -} +}; process.on('SIGINT', onProcessEnd); process.on('exit', onProcessEnd); @@ -151,23 +150,20 @@ async function getIndexJson(url) { const indexJsonUrl = new URL('index.json', url).toString(); const storiesJsonUrl = new URL('stories.json', url).toString(); - const [indexRes, storiesRes] = await Promise.all([ - fetch(indexJsonUrl), - fetch(storiesJsonUrl) - ]); + const [indexRes, storiesRes] = await Promise.all([fetch(indexJsonUrl), fetch(storiesJsonUrl)]); if (indexRes.ok) { try { const json = await indexRes.text(); return JSON.parse(json); - } catch (err) { } + } catch (err) {} } - if(storiesRes.ok) { + if (storiesRes.ok) { try { const json = await storiesRes.text(); return JSON.parse(json); - } catch (err) { } + } catch (err) {} } throw new Error(dedent` @@ -187,7 +183,7 @@ async function getIndexJson(url) { async function getIndexTempDir(url) { let tmpDir; try { - const indexJson = await getIndexJson(url) + const indexJson = await getIndexJson(url); const titleIdToTest = transformPlaywrightJson(indexJson); tmpDir = tempy.directory(); @@ -236,7 +232,7 @@ const main = async () => { const rawTargetURL = process.env.TARGET_URL || runnerOptions.url || 'http://localhost:6006'; await checkStorybook(rawTargetURL); - const targetURL = sanitizeURL(rawTargetURL) + const targetURL = sanitizeURL(rawTargetURL); process.env.TARGET_URL = targetURL; diff --git a/package.json b/package.json index 1b9f8cbf..52d5581e 100644 --- a/package.json +++ b/package.json @@ -42,12 +42,15 @@ "build-storybook": "build-storybook", "release": "yarn build && auto shipit", "test-storybook": "node bin/test-storybook.js", + "test-storybook:failures": "SKIP_SNAPSHOTS=true TEST_FAILURES=1 yarn test-storybook --json --outputFile test-results.json", "test-storybook:no-cache": "yarn test-storybook --no-cache", "test-storybook:json": "yarn test-storybook --stories-json", "test-storybook:ci": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook\"", + "test-storybook:ci-failures": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"TEST_FAILURES=1 yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:failures\"", "test-storybook:ci-coverage": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook --coverage\"", "test-storybook:ci-json": "concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"yarn build-storybook --quiet && npx http-server storybook-static --port 6006 --silent\" \"wait-on tcp:6006 && yarn test-storybook:json\"", - "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js" + "generate-dynamic-stories": "node scripts/generate-dynamic-stories.js", + "prepare": "husky install" }, "bin": { "test-storybook": "./bin/test-storybook.js" @@ -79,8 +82,10 @@ "babel-loader": "^8.1.0", "babel-plugin-istanbul": "^6.1.1", "concurrently": "^7.0.0", + "husky": "^8.0.0", "jest": "^27.0.6", "jest-image-snapshot": "^4.5.1", + "lint-staged": "^13.0.3", "prettier": "^2.3.1", "prop-types": "^15.7.2", "react": "^17.0.1", @@ -90,6 +95,9 @@ "typescript": "^4.2.4", "wait-on": "^6.0.0" }, + "lint-staged": { + "*.{ts,js,tsx,jsx,css,md}": "prettier --write" + }, "publishConfig": { "access": "public" }, diff --git a/playwright/custom-environment.js b/playwright/custom-environment.js index 5111e0a5..eb2457d9 100644 --- a/playwright/custom-environment.js +++ b/playwright/custom-environment.js @@ -6,7 +6,7 @@ const PlaywrightEnvironment = require('jest-playwright-preset/lib/PlaywrightEnvi class CustomEnvironment extends PlaywrightEnvironment { async setup() { await super.setup(); - await setupPage(this.global.page) + await setupPage(this.global.page); } async teardown() { diff --git a/playwright/jest-setup.js b/playwright/jest-setup.js index 9530720a..bf0d2bb4 100644 --- a/playwright/jest-setup.js +++ b/playwright/jest-setup.js @@ -13,4 +13,4 @@ if (testRunnerConfig) { } } -global.__sbCollectCoverage = process.env.STORYBOOK_COLLECT_COVERAGE === 'true' \ No newline at end of file +global.__sbCollectCoverage = process.env.STORYBOOK_COLLECT_COVERAGE === 'true'; diff --git a/playwright/test-runner-jest.config.js b/playwright/test-runner-jest.config.js index 80d3e57e..e4d7a201 100644 --- a/playwright/test-runner-jest.config.js +++ b/playwright/test-runner-jest.config.js @@ -6,4 +6,4 @@ module.exports = { /** Add your own overrides below * @see https://jestjs.io/docs/configuration */ -} \ No newline at end of file +}; diff --git a/playwright/transform.js b/playwright/transform.js index ed47eb16..a53fcced 100644 --- a/playwright/transform.js +++ b/playwright/transform.js @@ -15,6 +15,7 @@ module.exports = { '@babel/preset-react', ], }); + return result ? result.code : src; }, }; diff --git a/src/setup-page.ts b/src/setup-page.ts index e4e30cc5..66f1899d 100644 --- a/src/setup-page.ts +++ b/src/setup-page.ts @@ -1,5 +1,7 @@ -//@ts-nocheck -const sanitizeURL = (url) => { +import type { Page } from 'playwright'; +import dedent from 'ts-dedent'; + +const sanitizeURL = (url: string) => { let finalURL = url; // prepend URL protocol if not there if (finalURL.indexOf('http://') === -1 && finalURL.indexOf('https://') === -1) { @@ -20,9 +22,10 @@ const sanitizeURL = (url) => { return finalURL; }; -export const setupPage = async (page) => { +export const setupPage = async (page: Page) => { const targetURL = new URL('iframe.html', process.env.TARGET_URL).toString(); const viewMode = process.env.VIEW_MODE || 'story'; + const isCoverageMode = process.env.STORYBOOK_COLLECT_COVERAGE === 'true'; const renderedEvent = viewMode === 'docs' ? 'docsRendered' : 'storyRendered'; const referenceURL = process.env.REFERENCE_URL && sanitizeURL(process.env.REFERENCE_URL); @@ -43,33 +46,90 @@ export const setupPage = async (page) => { } throw err; - }); // FIXME: configure + }); + + if (isCoverageMode) { + const isCoverageSetupCorrectly = await page.evaluate(() => '__coverage__' in window); + if (!isCoverageSetupCorrectly) { + throw new Error( + dedent` + [Test runner] An error occurred when evaluating code coverage: + The code in Storybook is not instrumented, which means the coverage setup is not correct. + More info: https://github.com/storybookjs/test-runner#setting-up-code-coverage + ` + ); + } + } // if we ever want to log something from the browser to node await page.exposeBinding('logToPage', (_, message) => console.log(message)); await page.addScriptTag({ content: ` + // colorizes the console output + const bold = (message) => \`\\u001b[1m\${message}\\u001b[22m\`; + const magenta = (message) => \`\\u001b[35m\${message}\\u001b[39m\`; + const blue = (message) => \`\\u001b[34m\${message}\\u001b[39m\`; + const red = (message) => \`\\u001b[31m\${message}\\u001b[39m\`; + const yellow = (message) => \`\\u001b[33m\${message}\\u001b[39m\`; + + // removes circular references from the object + function serializer(replacer, cycleReplacer) { + let stack = [], + keys = []; + + if (cycleReplacer == null) + cycleReplacer = function (_key, value) { + if (stack[0] === value) return '[Circular]'; + return '[Circular ~.' + keys.slice(0, stack.indexOf(value)).join('.') + ']'; + }; + + return function (key, value) { + if (stack.length > 0) { + let thisPos = stack.indexOf(this); + ~thisPos ? stack.splice(thisPos + 1) : stack.push(this); + ~thisPos ? keys.splice(thisPos, Infinity, key) : keys.push(key); + if (~stack.indexOf(value)) value = cycleReplacer.call(this, key, value); + } else { + stack.push(value); + } + + return replacer == null ? value : replacer.call(this, key, value); + }; + } + + function safeStringify(obj, replacer, spaces, cycleReplacer) { + return JSON.stringify(obj, serializer(replacer, cycleReplacer), spaces); + } + + function composeMessage(args) { + if (typeof args === 'undefined') return "undefined"; + if (typeof args === 'string') return args; + return safeStringify(args); + } + function truncate(input, limit) { if (input.length > limit) { return input.substring(0, limit) + '…'; } return input; } - + class StorybookTestRunnerError extends Error { - constructor(storyId, errorMessage) { + constructor(storyId, errorMessage, logs) { super(errorMessage); this.name = 'StorybookTestRunnerError'; const storyUrl = \`${referenceURL || targetURL}?path=/story/\${storyId}\`; const finalStoryUrl = \`\${storyUrl}&addonPanel=storybook/interactions/panel\`; + const separator = '\\n\\n--------------------------------------------------'; + const extraLogs = logs.length > 0 ? separator + "\\n\\nBrowser logs:\\n\\n"+ logs.join('\\n\\n') : ''; - this.message = \`\nAn error occurred in the following story. Access the link for full output:\n\${finalStoryUrl}\n\nMessage:\n \${truncate(errorMessage,${debugPrintLimit})}\`; + this.message = \`\nAn error occurred in the following story. Access the link for full output:\n\${finalStoryUrl}\n\nMessage:\n \${truncate(errorMessage,${debugPrintLimit})}\n\${extraLogs}\`; } } - async function __throwError(storyId, errorMessage) { - throw new StorybookTestRunnerError(storyId, errorMessage); + async function __throwError(storyId, errorMessage, logs) { + throw new StorybookTestRunnerError(storyId, errorMessage, logs); } async function __waitForElement(selector) { @@ -118,18 +178,45 @@ export const setupPage = async (page) => { 'The test runner could not access the Storybook channel. Are you sure the Storybook is running correctly in that URL?' ); } + + // collect logs to show upon test error + let logs = []; + + const spyOnConsole = (method, name) => { + const originalFn = console[method]; + return function () { + const message = [...arguments].map(composeMessage).join(', '); + const prefix = \`\${bold(name)}: \`; + logs.push(prefix + message); + originalFn.apply(console, arguments); + }; + }; + + // console methods + color function for their prefix + const spiedMethods = { + log: blue, + warn: yellow, + error: red, + trace: magenta, + group: magenta, + groupCollapsed: magenta, + } + + Object.entries(spiedMethods).forEach(([method, color]) => { + console[method] = spyOnConsole(method, color(method)) + }) return new Promise((resolve, reject) => { channel.on('${renderedEvent}', () => resolve(document.getElementById('root'))); channel.on('storyUnchanged', () => resolve(document.getElementById('root'))); channel.on('storyErrored', ({ description }) => reject( - new StorybookTestRunnerError(storyId, description)) + new StorybookTestRunnerError(storyId, description, logs)) ); channel.on('storyThrewException', (error) => reject( - new StorybookTestRunnerError(storyId, error.message)) + new StorybookTestRunnerError(storyId, error.message, logs)) ); channel.on('storyMissing', (id) => id === storyId && reject( - new StorybookTestRunnerError(storyId, 'The story was missing when trying to access it.')) + new StorybookTestRunnerError(storyId, 'The story was missing when trying to access it.', logs)) ); channel.emit('setCurrentStory', { storyId, viewMode: '${viewMode}' }); diff --git a/src/util/getParsedCliOptions.ts b/src/util/getParsedCliOptions.ts index 04013486..54c89e7e 100644 --- a/src/util/getParsedCliOptions.ts +++ b/src/util/getParsedCliOptions.ts @@ -18,11 +18,7 @@ export const getParsedCliOptions = () => { '.storybook' ) .option('--watch', 'Watch files for changes and rerun tests related to changed files', false) - .option( - '--watchAll', - 'Watch files for changes and rerun all tests when something changes', - false - ) + .option('--watchAll', 'Watch files for changes and rerun all tests when something changes') .option( '--browsers ', 'Define browsers to run tests in. Could be one or multiple of: chromium, firefox, webkit', @@ -44,6 +40,14 @@ export const getParsedCliOptions = () => { '-u, --updateSnapshot', 'Use this flag to re-record every snapshot that fails during this test run' ) + .option( + '--json', + 'Prints the test results in JSON. This mode will send all other test output and user messages to stderr.' + ) + .option( + '--outputFile', + 'Write test results to a file when the --json option is also specified.' + ) .option( '--coverage', 'Indicates that test coverage information should be collected and reported in the output' diff --git a/stories/atoms/Button.js b/stories/atoms/Button.jsx similarity index 68% rename from stories/atoms/Button.js rename to stories/atoms/Button.jsx index 19c96568..15dde392 100644 --- a/stories/atoms/Button.js +++ b/stories/atoms/Button.jsx @@ -1,20 +1,16 @@ -import React from "react"; -import PropTypes from "prop-types"; -import "./button.css"; +import React from 'react'; +import PropTypes from 'prop-types'; +import './button.css'; /** * Primary UI component for user interaction */ export const Button = ({ primary, backgroundColor, size, label, ...props }) => { - const mode = primary - ? "storybook-button--primary" - : "storybook-button--secondary"; + const mode = primary ? 'storybook-button--primary' : 'storybook-button--secondary'; return (