From fd098bd53671698f99d97371f06743b3957cd9af Mon Sep 17 00:00:00 2001
From: Yann Braga <yannbf@gmail.com>
Date: Thu, 18 Aug 2022 12:05:17 +0200
Subject: [PATCH] add failure test to nightly check [skip-release]

---
 .github/workflows/nightly.yml | 95 +++++++++++++++++++++++++++++++++--
 .storybook/test-runner.ts     |  5 ++
 2 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index d347e0cb..7d97c5ca 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -3,7 +3,7 @@ name: Nightly checks
 # runs every day at midnight
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: '0 0 * * *'
   workflow_dispatch:
   # To test fixes on push rather than wait for the scheduling
   push:
@@ -11,7 +11,7 @@ on:
       - fix/nightly
 
 jobs:
-  test_storybook_prerelease:
+  assert_test_runner:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -37,7 +37,6 @@ jobs:
       - name: Fix local @storybook/csf version
         run: |
           yarn add @storybook/csf@0.0.2--canary.4566f4d.1
-
       - name: Run test runner
         uses: mathiasvr/command-output@v1
         id: tests
@@ -57,7 +56,7 @@ jobs:
         id: slack
         uses: slackapi/slack-github-action@v1.19.0
         with:
-          channel-id: "${{ secrets.SLACK_CHANNEL_ID }}"
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
           payload: |
             {
               "blocks": [
@@ -97,3 +96,91 @@ jobs:
             }
         env:
           SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+  assert_test_runner_failures:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+
+      - name: Install dependencies
+        uses: bahmutov/npm-install@v1
+
+      - name: Get current version of Storybook
+        run: |
+          echo "prev_sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "prev_sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Upgrade to storybook@future
+        run: |
+          npx storybook@future upgrade --prerelease --yes
+
+      # TODO: This should not be necessary once @storybook/csf is properly updated
+      - name: Fix local @storybook/csf version
+        run: |
+          yarn add @storybook/csf@0.0.2--canary.4566f4d.1
+      - name: Run test runner and expect failure
+        uses: mathiasvr/command-output@v1
+        id: tests
+        with:
+          run: |
+            yarn build
+            yarn test-storybook:ci
+        env:
+          TEST_FAILURES: true
+          SKIP_SNAPSHOTS: true
+
+      - name: Get prerelease version of Storybook
+        if: ${{ success() }}
+        run: |
+          echo "sb_version=$(yarn list @storybook/react --depth=0 2> /dev/null | grep @storybook/react | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+          echo "sb_csf_version=$(yarn list @storybook/csf --depth=0 2> /dev/null | grep @storybook/csf | awk -F'@' '{print $3}')" >> $GITHUB_ENV
+
+      - name: Report incoming errors
+        if: ${{ success() }}
+        id: slack
+        uses: slackapi/slack-github-action@v1.19.0
+        with:
+          channel-id: '${{ secrets.SLACK_CHANNEL_ID }}'
+          payload: |
+            {
+              "blocks": [
+                {
+                  "type": "header",
+                  "text": {
+                    "type": "plain_text",
+                    "text": ":storybook: :runner: [Test Runner] The Nightly check for **failures** has passed :thinking_face:",
+                    "emoji": true
+                  }
+                },
+                {
+                  "type": "section",
+                  "fields": [
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/react version:*\n${{ env.prev_sb_version }} >> ${{ env.sb_version }}"
+                    },
+                    {
+                      "type": "mrkdwn",
+                      "text": "*@storybook/csf version:*\n${{ env.prev_sb_csf_version }} >> ${{ env.sb_csf_version }}"
+                    }
+                  ],
+                  "accessory": {
+                    "type": "button",
+                    "text": {
+                      "type": "plain_text",
+                      "text": "View failure",
+                      "emoji": true
+                    },
+                    "value": "view_failure",
+                    "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                    "action_id": "button-action"
+                  }
+                }
+              ]
+            }
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
diff --git a/.storybook/test-runner.ts b/.storybook/test-runner.ts
index 76d89258..c36b5b0d 100644
--- a/.storybook/test-runner.ts
+++ b/.storybook/test-runner.ts
@@ -4,6 +4,7 @@ import type { TestRunnerConfig } from '../dist/ts';
 
 const snapshotsDir = process.env.SNAPSHOTS_DIR || '__snapshots__';
 const customSnapshotsDir = `${process.cwd()}/${snapshotsDir}`;
+const skipSnapshots = process.env.SKIP_SNAPSHOTS === 'true';
 
 const config: TestRunnerConfig = {
   setup() {
@@ -17,6 +18,10 @@ const config: TestRunnerConfig = {
       return;
     }
 
+    if (skipSnapshots) {
+      return;
+    }
+
     // Visual snapshot tests
     const image = await page.screenshot({ fullPage: true });
     expect(image).toMatchImageSnapshot({