test: improve ci e2e setup to include multiple browsers, optimize tra…

…ce capturing

.github/workflows/ci.yml

@@ -10,9 +10,17 @@ env:
   TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
   TURBO_TEAM: svelte
 
+# cancel in-progress runs on new commits to same PR (gitub.event.number)
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number || github.sha }}
+  cancel-in-progress: true
+
 jobs:
   Lint:
     runs-on: ubuntu-latest
+    env:
+      # not needed for linting
+      PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"
     steps:
       - uses: actions/checkout@v3
       - uses: pnpm/action-setup@v2.2.1
@@ -29,12 +37,23 @@ jobs:
       fail-fast: false
       matrix:
         node-version: [16]
-        os: [ubuntu-latest, macOS-latest, windows-2019]
+        os: [ubuntu-latest , windows-2019]
+        e2e-browser: 'chromium'
         include:
+          - node-version: 16
+            os: ubuntu-latest
+            e2e-browser: 'firefox'
+          - node-version: 16
+            os: macOS-latest
+            e2e-browser: 'safari'
           - node-version: 18
             os: ubuntu-latest
+            e2e-browser: 'chromium'
     env:
       TURBO_CACHE_KEY: ${{ matrix.os }}-${{ matrix.node-version }}
+      # Install playwright's binray under node_modules so it will be cached together
+      PLAYWRIGHT_BROWSERS_PATH: "0"
+      KIT_E2E_BROWSER: ${{matrix.e2e-browser}}
     steps:
       - run: git config --global core.autocrlf false
       - uses: actions/checkout@v3
@@ -44,11 +63,12 @@ jobs:
           node-version: ${{ matrix.node-version }}
           cache: pnpm
       - run: pnpm install --frozen-lockfile
+      - run: pnpm playwright install
       - run: pnpm test
       - name: archive test results
         if: failure()
         shell: bash
-        run: find packages -type d -name test-results -not -empty | tar -czf test-results.tar.gz --exclude='*-retry*' --files-from=-
+        run: find packages -type d -name test-results -not -empty | tar -czf test-results.tar.gz --exclude='*-retry2*' --exclude='*-retry3*' --exclude='*-retry4*' --exclude='*-retry5*' --files-from=-
       - name: Upload failed tests screenshots
         if: failure()
         uses: actions/upload-artifact@v3

packages/kit/test/utils.js

@@ -1,7 +1,7 @@
 import fs from 'fs';
 import http from 'http';
 import * as ports from 'port-authority';
-import { test as base } from '@playwright/test';
+import { test as base, devices } from '@playwright/test';
 
 export const test = base.extend({
 	// @ts-expect-error
@@ -135,6 +135,18 @@ export const test = base.extend({
 		use(read_errors);
 	}
 });
+const test_browser = process.env.KIT_E2E_BROWSER ?? 'chromium';
+const known_devices = {
+	chromium: devices['Desktop Chrome'],
+	firefox: devices['Desktop Firefox'],
+	safari: devices['Desktop Safari']
+};
+
+const test_browser_device = known_devices[test_browser];
+
+if (!test_browser_device) {
+	throw new Error(`invalid test browser specified: KIT_E2E_BROWSER=${process.env.KIT_E2E_BROWSER}. Allowed values: ${Object.keys(known_devices).join(', ')}`);
+}
 
 /** @type {import('@playwright/test').PlaywrightTestConfig} */
 export const config = {
@@ -148,24 +160,22 @@ export const config = {
 	retries: process.env.CI ? 5 : 0,
 	projects: [
 		{
-			name: `${process.env.DEV ? 'dev' : 'build'}+js`,
+			name: `${test_browser}-${process.env.DEV ? 'dev' : 'build'}+js`,
 			use: {
 				javaScriptEnabled: true
 			}
 		},
 		{
-			name: `${process.env.DEV ? 'dev' : 'build'}-js`,
+			name: `${test_browser}-${process.env.DEV ? 'dev' : 'build'}-js`,
 			use: {
 				javaScriptEnabled: false
 			}
 		}
 	],
 	use: {
+		...test_browser_device,
 		screenshot: 'only-on-failure',
-		trace: 'retain-on-failure',
-		// use stable chrome from host OS instead of downloading one
-		// see https://playwright.dev/docs/browsers#google-chrome--microsoft-edge
-		channel: 'chrome'
+		trace: process.env.KIT_E2E_TRACE ? 'on' : 'on-first-retry'
 	},
 	workers: process.env.CI ? 2 : undefined
 };