Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@kwonoj
kwonoj committed May 22, 2023
1 parent 789f253 commit f897bed
Show file tree
Hide file tree
Showing 4 changed files with 149 additions and 13 deletions.
120 changes: 116 additions & 4 deletions .github/workflows/build_test_deploy.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -812,9 +812,8 @@ jobs:
- run: xvfb-run node run-tests.js test/integration/with-electron/test/index.test.js
if: ${{needs.build.outputs.docsChange == 'nope'}}

# A job to run sets of tests with turbopack enabled. These tests are considered as `stable`,
# that running with turbopack should always pass.
testTurbopack:
# A job to run sets of devlopment tests with turbopack enabled.
testDevTurbopack:
name: Test Development (Turbopack)
runs-on: ubuntu-latest
needs: [build, build-native-test]
Expand All @@ -832,6 +831,10 @@ jobs:

strategy:
fail-fast: false
matrix:
node: [18]
# Currently turbopack tests does not utilize groups yet
group: [1]
steps:
- run: echo "${{needs.build.outputs.docsChange}}"

Expand All @@ -853,10 +856,119 @@ jobs:
name: next-swc-test-binary
path: packages/next-swc/native

- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -c 1 >> /proc/1/fd/1"
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings --timings >> /proc/1/fd/1"
name: Run test/development
if: ${{needs.build.outputs.docsChange == 'nope'}}

testDevE2ETurbopack:
name: Test Development (E2E) (Turbopack)
runs-on: ubuntu-latest
needs: [build, build-native-test]
timeout-minutes: 35
env:
NEXT_TELEMETRY_DISABLED: 1
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Enabling backtrace will makes snapshot tests fail
RUST_BACKTRACE: 0
# Path to the custom next-swc bindings.
NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
# Glob pattern to run specific tests with --turbo.
NEXT_DEV_TEST_GLOB: '*'
NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
strategy:
fail-fast: false
matrix:
node: [18]
# Currently turbopack tests does not utilize groups yet
group: [1]
steps:
- run: echo "${{needs.build.outputs.docsChange}}"

# https://github.com/actions/virtual-environments/issues/1187
- name: tune linux network
run: sudo ethtool -K eth0 tx off rx off

- uses: actions/cache@v3
timeout-minutes: 5
if: ${{needs.build.outputs.docsChange == 'nope'}}
id: restore-build
with:
path: ./*
key: ${{ github.sha }}-${{ github.run_number }}

- uses: actions/download-artifact@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
name: next-swc-test-binary
path: packages/next-swc/native

- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings >> /proc/1/fd/1"
name: Run test/e2e (dev)
if: ${{needs.build.outputs.docsChange == 'nope'}}

- name: Upload test trace
if: always()
uses: actions/upload-artifact@v3
with:
name: test-trace
if-no-files-found: ignore
retention-days: 2
path: |
test/traces
testIntegrationTurbopack:
name: Test Integration (Turbopack)
runs-on: ubuntu-latest
needs: [build, build-native-test]
timeout-minutes: 35
env:
NEXT_TELEMETRY_DISABLED: 1
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
# Enabling backtrace will makes snapshot tests fail
RUST_BACKTRACE: 0
# Path to the custom next-swc bindings.
NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
# Glob pattern to run specific tests with --turbo.
NEXT_DEV_TEST_GLOB: '*'
NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
strategy:
fail-fast: false
matrix:
group: [1]
steps:
- run: echo "${{needs.build.outputs.docsChange}}"

# https://github.com/actions/virtual-environments/issues/1187
- name: tune linux network
run: sudo ethtool -K eth0 tx off rx off

- uses: actions/cache@v3
timeout-minutes: 5
if: ${{needs.build.outputs.docsChange == 'nope'}}
id: restore-build
with:
path: ./*
key: ${{ github.sha }}-${{ github.run_number }}

- uses: actions/download-artifact@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
name: next-swc-test-binary
path: packages/next-swc/native

- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings >> /proc/1/fd/1"
if: ${{needs.build.outputs.docsChange == 'nope'}}

- name: Upload test trace
if: always()
uses: actions/upload-artifact@v3
with:
name: test-trace
if-no-files-found: ignore
retention-days: 2
path: |
test/traces
testsPass:
name: thank you, next
runs-on: ubuntu-latest
Expand Down
5 changes: 3 additions & 2 deletions packages/next-swc/crates/next-dev-tests/sync-tests-manifest.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,11 @@ const path = require('path')

const generateManifest = (enabledTests, disabledTests) => `
// Tests that are currently enabled with Turbopack in CI.
// Add new test when Turbopack updates to fix / implement a feature.
// This list is not actively used, more of high level pictures of what tests are enabled.
const enabledTests = ${enabledTests}
// Tests that are currently disabled with Turbopack in CI.
// Any tests not listed in here are assumed to be enabled.
const disabledTests = ${disabledTests}
module.exports = {
Expand All @@ -30,7 +31,7 @@ const main = async () => {
enabledTests = manifest.enabledTests
disabledTests = manifest.disabledTests
} else {
throw new Error('a')
throw new Error('Manifest should exists')
}

// Collect all test files
Expand Down
6 changes: 5 additions & 1 deletion packages/next-swc/crates/next-dev-tests/tests-manifest.js
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// Tests that are currently enabled with Turbopack in CI.
// Add new test when Turbopack updates to fix / implement a feature.
// This list is not actively used, more of high level pictures of what tests are enabled.
const enabledTests = [
'test/development/acceptance-app/dynamic-error.test.ts',
'test/development/basic/legacy-decorators.test.ts',
Expand All @@ -9,6 +9,7 @@ const enabledTests = [
]

// Tests that are currently disabled with Turbopack in CI.
// Any tests not listed in here are assumed to be enabled.
const disabledTests = [
'test/development/acceptance-app/app-hmr-changes.test.ts',
'test/development/acceptance-app/component-stack.test.ts',
Expand Down Expand Up @@ -76,6 +77,7 @@ const disabledTests = [
'test/e2e/404-page-router/index.test.ts',
'test/e2e/app-dir-legacy-edge-runtime-config/index.test.ts',
'test/e2e/app-dir/_allow-underscored-root-directory/_allow-underscored-root-directory.test.ts',
'test/e2e/app-dir/actions/app-action-export.test.ts',
'test/e2e/app-dir/actions/app-action.test.ts',
'test/e2e/app-dir/app-a11y/index.test.ts',
'test/e2e/app-dir/app-alias/app-alias.test.ts',
Expand Down Expand Up @@ -108,6 +110,7 @@ const disabledTests = [
'test/e2e/app-dir/dynamic-href/dynamic-href.test.ts',
'test/e2e/app-dir/dynamic/dynamic.test.ts',
'test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts',
'test/e2e/app-dir/error-boundary-and-not-found-linking/error-boundary-and-not-found-linking.test.ts',
'test/e2e/app-dir/front-redirect-issue/front-redirect-issue.test.ts',
'test/e2e/app-dir/global-error/global-error.test.ts',
'test/e2e/app-dir/hooks/hooks.test.ts',
Expand Down Expand Up @@ -672,6 +675,7 @@ const disabledTests = [
'test/unit/esm-interpolate/esm-interpolate.test.tsx',
'test/unit/find-config.test.ts',
'test/unit/find-page-file.test.ts',
'test/unit/get-module-build-info.test.ts',
'test/unit/get-node-options-without-inspect.test.ts',
'test/unit/get-page-static-infos.test.ts',
'test/unit/get-project-dir.test.ts',
Expand Down
31 changes: 25 additions & 6 deletions run-tests.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ const { createNextInstall } = require('./test/lib/create-next-install')
const glob = promisify(_glob)
const exec = promisify(execOrig)

// Try to read an external array-based json to filter tests to be executed.
// Try to read an external array-based json to filter tests to be allowed / or disallowed.
// If process.argv contains a test to be executed, this'll append it to the list.
const externalTestsFilterLists = process.env.NEXT_EXTERNAL_TESTS_FILTERS
? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS).enabledTests
: []
? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS)
: { enabledTests: [], disabledTests: [] }
const timings = []
const DEFAULT_NUM_RETRIES = os.platform() === 'win32' ? 2 : 1
const DEFAULT_CONCURRENCY = 2
Expand Down Expand Up @@ -143,9 +143,7 @@ async function main() {

console.log('Running tests with concurrency:', concurrency)

let tests = process.argv
.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
.concat(externalTestsFilterLists)
let tests = process.argv.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
let prevTimings

if (tests.length === 0) {
Expand Down Expand Up @@ -190,6 +188,27 @@ async function main() {
}
}

// If there are external manifest contains list of tests, apply it to the test lists
// Currently, we expect manifest should include named export to the array `disabledTests`,
// which we'll explicitly filter out from the test list.
console.log('pppp', tests.length)
if (externalTestsFilterLists.disabledTests.length > 0) {
tests = tests.filter(
(test) =>
!externalTestsFilterLists.disabledTests.some((disabled) =>
disabled.includes(test)
)
)

console.log('x', {
disabled: externalTestsFilterLists.disabledTests,
tests,
})
console.log('a', externalTestsFilterLists.disabledTests.length)

console.log('pppp2', tests)
}

let testNames = [
...new Set(
tests.map((f) => {
Expand Down

0 comments on commit f897bed

Please sign in to comment.