Cover 1st-party examples with tests #45270
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| # This file duplicates bunch of things from build_test_deploy | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| is_dispatched: | ||
| description: 'Leave this option enabled' | ||
| required: true | ||
| default: true | ||
| type: boolean | ||
| schedule: | ||
| - cron: '0 */4 * * *' | ||
|
|
||
| name: Test examples | ||
|
|
||
| env: | ||
| PNPM_VERSION: 7.24.3 | ||
|
|
||
| jobs: | ||
| testExamples: | ||
| # Don't execute using cron on forks | ||
| if: (github.repository == 'vercel/next.js') || (inputs.is_dispatched == true) | ||
| name: Test Examples | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 120 | ||
| env: | ||
| NEXT_TELEMETRY_DISABLED: 1 | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| node: [16, 18] | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| with: | ||
| fetch-depth: 25 | ||
| # https://github.com/actions/virtual-environments/issues/1187 | ||
| - name: tune linux network | ||
| run: sudo ethtool -K eth0 tx off rx off | ||
|
|
||
| - name: Setup node | ||
| uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: 16 | ||
| check-latest: true | ||
|
|
||
| - run: npm i -g pnpm@${PNPM_VERSION} | ||
|
|
||
| - run: pnpm install | ||
| - run: pnpm build | ||
|
|
||
| - run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-focal /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_TEST_JOB=1 NEXT_TEST_MODE=start xvfb-run node run-tests.js --type examples >> /proc/1/fd/1" | ||
| name: Run test/examples |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| import { createNextDescribe } from 'e2e-utils' | ||
| import path from 'path' | ||
| import fs from 'fs-extra' | ||
|
|
||
| const testedExamples = [ | ||
| // Internal features | ||
| 'active-class-name', | ||
| 'amp', | ||
| 'amp-first', | ||
| 'amp-story', | ||
| 'api-routes', | ||
| 'api-routes-cors', | ||
| 'api-routes-middleware', | ||
| 'api-routes-rate-limit', | ||
| 'api-routes-rest', | ||
| 'app-dir-i18n-routing', | ||
| 'app-dir-mdx', | ||
| 'basic-css', | ||
| 'basic-export', | ||
| 'blog', | ||
| 'blog-starter', | ||
| 'catch-all-routes', | ||
| 'custom-routes-proxying', | ||
| 'custom-server', | ||
| 'data-fetch', | ||
| 'dynamic-routing', | ||
| 'environment-variables', | ||
| 'fast-refresh-demo', | ||
| 'head-elements', | ||
| 'headers', | ||
| 'hello-world', | ||
| 'hello-world-esm', | ||
| 'i18n-routing', | ||
| 'image-component', | ||
| 'image-legacy-component', | ||
| 'layout-component', | ||
| 'middleware', | ||
| 'middleware-matcher', | ||
| 'modularize-imports', | ||
| 'nested-components', | ||
| 'next-css', | ||
| 'next-forms', | ||
| 'progressive-render', | ||
| 'redirects', | ||
| 'remove-console', | ||
| 'reproduction-template', | ||
| 'rewrites', | ||
| 'script-component', | ||
| 'ssr-caching', | ||
| 'styled-jsx-with-csp', | ||
| 'svg-components', | ||
| 'using-router', | ||
| 'with-absolute-imports', | ||
| 'with-app-layout', | ||
| 'with-context-api', | ||
| 'with-env-from-next-config-js', | ||
| 'with-loading', | ||
| 'with-shallow-routing', | ||
| 'with-sitemap', | ||
| 'with-typescript', | ||
| 'with-typescript-types', | ||
| 'with-web-worker', | ||
| 'with-webassembly', | ||
|
|
||
| // Library integrations that we can't break | ||
| 'with-jest', | ||
| 'with-jest-babel', | ||
| 'with-mdx', | ||
| 'with-mdx-remote', | ||
| 'with-tailwindcss', | ||
| 'with-turbopack', | ||
| 'with-vercel-fetch', | ||
| ] | ||
|
|
||
| const testExample = (example) => { | ||
| const exampleFiles = path.join(__dirname, '..', '..', 'examples', example) | ||
|
|
||
| const packageJson = fs.readJsonSync(path.join(exampleFiles, 'package.json')) | ||
| describe(`example '${example}'`, () => { | ||
| // If there is an issue during a build, jest won't tell us which example caused it | ||
| // we need to log it ourselfs | ||
| beforeAll(() => { | ||
| require('console').log(`Running example '${example}'`) | ||
| }) | ||
| createNextDescribe( | ||
| `example '${example}'`, | ||
|
There was a problem hiding this comment. Is the There was a problem hiding this comment. Yes, See the comment. We need to fire |
||
| { | ||
| files: exampleFiles, | ||
| dependencies: { | ||
| // We need to make sure that these default dependencies are not installed by default | ||
| // for our examples to ensure that they have all their dependencies in package.json | ||
| '@types/node': undefined, | ||
| '@types/react': undefined, | ||
| next: undefined, | ||
| react: undefined, | ||
| 'react-dom': undefined, | ||
| typescript: undefined, | ||
| ...packageJson.dependencies, | ||
| ...packageJson.devDependencies, | ||
| }, | ||
| }, | ||
| () => { | ||
| it('builds', () => {}) | ||
| } | ||
| ) | ||
| }) | ||
| } | ||
|
|
||
| testedExamples.forEach(testExample) | ||
jankaifer marked this conversation as resolved.
Show resolved
Hide resolved
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Instead of using an allowlist, we might want to change this to an ignorelist so that new examples are automatically tested.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since we are not running in normal CI it's actually possible now 👍
We probably should run all examples and we will see in the CI exactly which ones are broken.
But I still think that we want to have a workflow like this with only 1st party examples that should be always green.
And if we start running CI on all examples we need to be extra cautious when adding new libraries because we will be installing that library with access to our CI secrets.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Will add this in Follow-up PR