Skip to content

Commit

Permalink
Run all dev tests with replay on retry (#41168)
Browse files Browse the repository at this point in the history 
<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change that you're making:
-->

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

@ijjk WIP here.

Co-authored-by: JJ Kasper <jj@jjsweb.site>
  • Loading branch information
@jaril @ijjk
jaril and ijjk committed Oct 19, 2022
1 parent d76d827 commit 008a962
Showing 1 changed file with 134 additions and 4 deletions.
138 changes: 134 additions & 4 deletions .github/workflows/build_test_deploy.yml
View file
Expand Up @@ -286,9 +286,25 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type development --timings -g ${{ matrix.group }}/2
name: Run test/development
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDev / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

- name: Upload test trace
if: always()
Expand Down Expand Up @@ -346,9 +362,25 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type development --timings -g ${{ matrix.group }}/2
name: Run test/development
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDevReact17 / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

- name: Upload test trace
if: always()
Expand Down Expand Up @@ -421,7 +453,6 @@ jobs:
NEXT_TEST_MODE: dev
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}
# DEBUG: pw:browser*

- uses: replayio/action-upload@v0.4.5
if: always()
Expand Down Expand Up @@ -487,9 +518,26 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: NEXT_TEST_MODE=dev node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
name: Run test/e2e (dev)
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDevE2EReact17 / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
NEXT_TEST_MODE: dev
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

- name: Upload test trace
if: always()
Expand Down Expand Up @@ -548,9 +596,25 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type production --timings -g ${{ matrix.group }}/2
name: Run test/production
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testProd / Group ${{ matrix.group }} / Node ${{ matrix.node }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

testProdReact17:
name: Test Production (react v17)
Expand Down Expand Up @@ -598,9 +662,25 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type production --timings -g ${{ matrix.group }}/2
name: Run test/production
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testProdReact17 / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

testProdE2E:
name: Test Production (E2E)
Expand Down Expand Up @@ -650,9 +730,26 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: NEXT_TEST_MODE=start node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
name: Run test/e2e (production)
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testProdE2E / Group ${{ matrix.group }} / Node ${{ matrix.node }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
NEXT_TEST_MODE: start
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

testProdE2EReact17:
name: Test Production (E2E) (react v17)
Expand Down Expand Up @@ -701,9 +798,26 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: NEXT_TEST_MODE=start node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
name: Run test/e2e (production)
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testProdE2EReact17 / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
NEXT_TEST_MODE: start
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

testIntegration:
name: Test Integration
Expand Down Expand Up @@ -774,8 +888,24 @@ jobs:
timeout-minutes: 10
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: npx @replayio/playwright install chromium
if: ${{needs.build.outputs.docsChange == 'nope'}}

- run: xvfb-run node run-tests.js --timings -g ${{ matrix.group }}/20
if: ${{needs.build.outputs.docsChange == 'nope'}}
env:
RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testIntegration / Group ${{ matrix.group }}
RECORD_ALL_CONTENT: 1
RECORD_REPLAY: 1
RECORD_REPLAY_TEST_METRICS: 1
RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}

- uses: replayio/action-upload@v0.4.5
if: always()
with:
api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
public: true
filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}

- name: Upload test trace
if: always()
Expand Down

0 comments on commit 008a962

Please sign in to comment.