test: fix e2e + add ecosystem-ci test script (#311)

* chore: add ecosystem-ci scripts

* chore: lint

* chore: lint

* chore: fix escape

* chore: handle package overrides

* chore: tweak

* fix: fix script

* fix: more imba

* fix: --no-frozen-lockfile

* fix: npm overrides

* chore: run e2e

* ci: fix ci

* ci: typo

* ci: fix cd?

* ci: tweak

* chore: simplify xvfb-run

* test: use playwright assertion

* chore: simplify xvfb-run

* chore: readme

.github/workflows/build.yml

@@ -29,19 +29,15 @@ jobs:
       - run: pnpm typecheck
       - run: pnpm lint
 
-      # see: https://github.com/microsoft/vscode-test/blob/main/sample/azure-pipelines.yml
-      - name: xvfb
+      - name: test
         run: |
-          /usr/bin/Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
-          echo ">>> Started xvfb"
+          xvfb-run pnpm test
 
-      - run: pnpm test
-        env:
-          DISPLAY: ':99'
-
-      - run: pnpm test-e2e --retry 2
-        env:
-          DISPLAY: ':99'
+      - name: test-e2e
+        run: |
+          pnpm -C samples/e2e i
+          npm -C samples/imba i
+          xvfb-run pnpm test-e2e --retry 2
 
       - uses: actions/upload-artifact@v3
         if: always()

package.json

@@ -107,6 +107,8 @@
     "test": "vscode-test",
     "test:watch": "vscode-test --watch-files src/**/*.ts --watch-files test/**/*.test.ts",
     "test-e2e": "vitest --root test-e2e",
+    "ecosystem-ci:build": "pnpm compile",
+    "ecosystem-ci:test": "tsx ./scripts/ecosystem-ci.mts",
     "typecheck": "tsc -b ./ ./test ./test-e2e",
     "lint": "eslint --cache .",
     "lint:fix": "eslint --cache --fix ."
@@ -115,6 +117,7 @@
     "@antfu/eslint-config": "^2.6.4",
     "@babel/parser": "^7.20.15",
     "@babel/types": "^7.20.7",
+    "@playwright/test": "^1.42.1",
     "@rauschma/stringio": "^1.4.0",
     "@types/fs-extra": "^9.0.13",
     "@types/glob": "^7.2.0",
@@ -135,6 +138,7 @@
     "chai": "^5.1.0",
     "changelogithub": "^0.13.3",
     "eslint": "^8.56.0",
+    "execa": "^8.0.1",
     "fast-glob": "^3.3.2",
     "flatted": "^3.2.9",
     "fs-extra": "^10.0.1",
@@ -146,7 +150,6 @@
     "mocha": "^10.3.0",
     "pathe": "^1.1.2",
     "picocolors": "^1.0.0",
-    "playwright": "^1.42.1",
     "prompts": "^2.4.2",
     "semver": "^7.3.5",
     "strip-ansi": "^7.1.0",

scripts/ecosystem-ci.mts

@@ -0,0 +1,46 @@
+import process from 'node:process'
+import fs from 'node:fs'
+import { $ as $_ } from 'execa'
+
+const $ = $_({ stdio: 'inherit', verbose: true })
+
+async function main() {
+  // setup pakcage overrides for samples used by test-e2e
+  const pkg = await readJson('package.json')
+  if (pkg.pnpm?.overrides) {
+    // pnpm
+    await editJson('samples/e2e/package.json', (pkg2) => {
+      pkg2.pnpm = pkg.pnpm
+      return pkg2
+    })
+    await $({ cwd: 'samples/e2e' })`pnpm i --no-frozen-lockfile`
+
+    // npm
+    await editJson('samples/imba/package.json', (pkg2) => {
+      pkg2.devDependencies.vitest = pkg.pnpm.overrides.vitest
+      pkg2.overrides = pkg.pnpm.overrides
+      return pkg2
+    })
+    await $({ cwd: 'samples/imba' })`npm i`
+  }
+
+  if (process.env.CI === 'true' && process.platform === 'linux') {
+    await $`xvfb-run pnpm test`
+    await $`xvfb-run pnpm test-e2e --retry 2`
+  }
+  else {
+    await $`pnpm test`
+    await $`pnpm test-e2e`
+  }
+}
+
+async function readJson(file: string) {
+  return JSON.parse(await fs.promises.readFile(file, 'utf-8'))
+}
+
+async function editJson(file: string, edit: (data: any) => any) {
+  const data = await readJson(file)
+  await fs.promises.writeFile(file, JSON.stringify(edit(data), null, 2))
+}
+
+main()

test-e2e/README.md

@@ -0,0 +1,13 @@
+# test-e2e
+
+To open playwright inspector, either run with `PWDEBUG`
+
+```sh
+PWDEBUG=1 pnpm test-e2e
+```
+
+or use `page.pause` inside a test code
+
+```ts
+await page.pase()
+```

test-e2e/basic.test.ts

@@ -1,4 +1,5 @@
 import { beforeAll } from 'vitest'
+import { expect } from '@playwright/test'
 import { test } from './helper'
 
 // Vitst extension doesn't work with CI flag
@@ -23,10 +24,10 @@ test('basic', async ({ launch }) => {
   await page.getByRole('button', { name: 'Run Tests' }).click()
 
   // check results
-  await page.locator(`[title*="pass.test.ts (Passed)"]`).click()
-  await page.locator(`[title*="fail.test.ts (Failed)"]`).click()
-  await page.locator(`[title*="mix.test.ts (Failed)"]`).click()
-  await page.locator(`[title*="3/7 tests passed"]`).click()
+  await expect(page.locator(`[title*="3/7 tests passed"]`)).toBeVisible()
+  await expect(page.locator(`[title*="pass.test.ts (Passed)"]`)).toBeVisible()
+  await expect(page.locator(`[title*="fail.test.ts (Failed)"]`)).toBeVisible()
+  await expect(page.locator(`[title*="mix.test.ts (Failed)"]`)).toBeVisible()
 })
 
 test('imba', async ({ launch }) => {
@@ -46,7 +47,8 @@ test('imba', async ({ launch }) => {
   await page.getByRole('button', { name: 'Run Tests' }).click()
 
   // check results
-  await page.locator(`[title*="basic.test.imba (Passed)"]`).click()
-  await page.locator(`[title*="utils.imba (Passed)"]`).click()
-  await page.locator(`[title*="counter.imba (Failed)"]`).click()
+  await expect(page.locator(`[title*="5/7 tests passed"]`)).toBeVisible()
+  await expect(page.locator(`[title*="basic.test.imba (Passed)"]`)).toBeVisible()
+  await expect(page.locator(`[title*="utils.imba (Passed)"]`)).toBeVisible()
+  await expect(page.locator(`[title*="counter.imba (Failed)"]`)).toBeVisible()
 })

test-e2e/helper.ts

@@ -2,8 +2,8 @@ import fs from 'node:fs'
 import os from 'node:os'
 import path from 'node:path'
 import { download } from '@vscode/test-electron'
-import { _electron } from 'playwright'
-import type { Page } from 'playwright'
+import { _electron } from '@playwright/test'
+import type { Page } from '@playwright/test'
 import { test as baseTest } from 'vitest'
 
 // based on