Improved testing options (#460)

* Improved testing options

* Review feedback

* Fix typo

* Slight tweaks

* Allow multiple browsers or metrics

* Add e2e test

* Test action

* Revert test

* Revert properly

* Chrome tests

* Inly unit tests for now

* Retry Chrome tests

* Comment

* Disable gpu

* Try some mroe options

* One more try

* Onre more, one more try

* Give up, leave to just unit tests

* Rename test to make clear

* Try MacOS

* Set screensize

* More tests

* Set better standard size

* Plit out tests

* Revert back to one file

* Fix flakey test

* More flakiness

* didn'\t need visibility change

* Machines

* Firefox to MacOS

* Make command line flags case insensitive

* Revert test change

* Add rAF

* Fixup CLS tests

* Fix one genuine 0

* Add await

* Clear beacons

* Add another await

* More gte to gt

* strictEquals

* Clean up

* See if equals now works

---------

Co-authored-by: Philip Walton <philip@philipwalton.com>

.github/workflows/lint.yml

@@ -8,10 +8,10 @@ on:
 jobs:
   lint:
     name: Lint Code Base
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: NPM install
         run: npm install
       - name: Run Prettier

.github/workflows/tests.yml

@@ -0,0 +1,66 @@
+name: Run tests
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+jobs:
+  unit-tests:
+    name: Run unit tests
+    # Doesn't require anything special so let's use ubuntu as more available
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: NPM install
+        run: npm install
+      - name: Build
+        run: npm run build
+      - name: Run unit tests
+        run: npm run test:unit
+  chrome-tests:
+    name: Run Chrome e2e tests
+    # Runs best on macos for CI as linux requires extra chrome flags
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: NPM install
+        run: npm install
+      - name: Build
+        run: npm run build
+      - name: Run server
+        run: npm run test:server &
+      - name: Run e2e tests for chrome
+        run: npm run test:e2e -- --browsers=chrome
+  firefox-tests:
+    name: Run Firefox e2e tests
+    # Runs best on macos for CI as linux requires extra setup
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: NPM install
+        run: npm install
+      - name: Build
+        run: npm run build
+      - name: Run server
+        run: npm run test:server &
+      - name: Run e2e tests for firefox
+        run: npm run test:e2e -- --browsers=firefox
+  safari-tests:
+    name: Run Safari e2e tests
+    # Requires macos
+    runs-on: macos-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+      - name: NPM install
+        run: npm install
+      - name: Build
+        run: npm run build
+      - name: Run server
+        run: npm run test:server &
+      - name: Run e2e tests for safari
+        run: npm run test:e2e -- --browsers=safari

.husky/pre-commit

@@ -2,3 +2,11 @@
 . "$(dirname -- "$0")/_/husky.sh"
 
 npm exec lint-staged
+
+grep -r "\.only(" test/e2e \
+  && echo "ERROR: found .only() use in test" && exit 1
+
+grep -r "browser\.debug(" test/e2e \
+  && echo "ERROR: found browser.debug() use in test" && exit 1
+
+exit 0

docs/contributing.md → CONTRIBUTING.md

@@ -22,6 +22,26 @@ use GitHub pull requests for this purpose. Consult
 [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
 information on using pull requests.
 
+## Testing
+
+To test the full suite run `npm run test`.
+
+To test a subset of browsers or metrics, run the following in separate terminals:
+
+- `npm run watch`
+- `npm run test:server`
+- `npm run test:e2e -- --browsesr=chrome --metrics=TTFB`
+
+The last command can be replaced as you see fit and include comma, separated values. For example:
+
+- `npm run test:e2e -- --browsesr=chrome,firefox --metrics=TTFB,LCP`
+
+To run an individual test, change `it('test name')` to `it.only('test name')`.
+
+You can also add `await browser.debug()` lines to the individual test files to pause execution, and press `CTRL+C` in the command line to continue the tests.
+
+See the https://webdriver.io/ for more information.
+
 ## Community Guidelines
 
 This project follows [Google's Open Source Community

package.json

@@ -164,7 +164,8 @@
     "rollup": "^4.9.1",
     "selenium-standalone": "^9.3.1",
     "typescript": "^5.3.3",
-    "wdio-chromedriver-service": "^8.1.1"
+    "wdio-chromedriver-service": "^8.1.1",
+    "yargs": "^17.7.2"
   },
   "lint-staged": {
     "**/*.{js,ts}": "eslint --fix --ignore-path .gitignore",

test/e2e/onCLS-test.js

@@ -31,6 +31,9 @@ describe('onCLS()', async function () {
   let browserSupportsCLS;
   before(async function () {
     browserSupportsCLS = await browserSupportsEntry('layout-shift');
+
+    // Set a standard screen size so thresholds are the same
+    browser.setWindowSize(1280, 1024);
   });
 
   beforeEach(async function () {
@@ -50,7 +53,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
 
     const [cls] = await getBeacons();
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -71,7 +74,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
 
     const [cls] = await getBeacons();
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -92,7 +95,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
 
     const [cls] = await getBeacons();
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -117,7 +120,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
 
     const [cls] = await getBeacons();
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -140,7 +143,7 @@ describe('onCLS()', async function () {
 
     const [cls1] = await getBeacons();
 
-    assert(cls1.value >= 0);
+    assert(cls1.value > 0);
     assert(cls1.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls1.name, 'CLS');
     assert.strictEqual(cls1.value, cls1.delta);
@@ -268,7 +271,7 @@ describe('onCLS()', async function () {
     assert.strictEqual(cls1.entries.length, 0);
     assert.match(cls1.navigationType, /navigate|reload/);
 
-    assert(cls2.value >= 0);
+    assert(cls2.value > 0);
     assert.strictEqual(cls2.name, 'CLS');
     assert.strictEqual(cls2.id, cls1.id);
     assert.strictEqual(cls2.value, cls1.delta + cls2.delta);
@@ -312,7 +315,7 @@ describe('onCLS()', async function () {
     assert.strictEqual(cls1.entries.length, 0);
     assert.match(cls1.navigationType, /navigate|reload/);
 
-    assert(cls2.value >= 0);
+    assert(cls2.value > 0);
     assert.strictEqual(cls2.name, 'CLS');
     assert.strictEqual(cls2.id, cls1.id);
     assert.strictEqual(cls2.value, cls1.delta + cls2.delta);
@@ -352,8 +355,8 @@ describe('onCLS()', async function () {
 
     const [cls1] = await getBeacons();
 
-    assert(cls1.value >= 0);
-    assert(cls1.delta >= 0);
+    assert(cls1.value > 0);
+    assert(cls1.delta > 0);
     assert(cls1.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls1.name, 'CLS');
     assert.strictEqual(cls1.value, cls1.delta);
@@ -399,7 +402,7 @@ describe('onCLS()', async function () {
     assert.strictEqual(cls1.entries.length, 0);
     assert.match(cls1.navigationType, /navigate|reload/);
 
-    assert(cls2.value >= 0);
+    assert(cls2.value > 0);
     assert.strictEqual(cls2.name, 'CLS');
     assert.strictEqual(cls2.id, cls1.id);
     assert.strictEqual(cls2.value, cls1.delta + cls2.delta);
@@ -450,7 +453,7 @@ describe('onCLS()', async function () {
 
     const [cls1] = await getBeacons();
 
-    assert(cls1.value >= 0);
+    assert(cls1.value > 0);
     assert(cls1.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls1.delta, cls1.value);
     assert.strictEqual(cls1.name, 'CLS');
@@ -467,7 +470,7 @@ describe('onCLS()', async function () {
 
     const [cls2] = await getBeacons();
 
-    assert(cls2.value >= 0);
+    assert(cls2.value > 0);
     assert(cls2.id.match(/^v4-\d+-\d+$/));
     assert(cls2.id !== cls1.id);
 
@@ -485,7 +488,7 @@ describe('onCLS()', async function () {
 
     const [cls3] = await getBeacons();
 
-    assert(cls3.value >= 0);
+    assert(cls3.value > 0);
     assert(cls3.id.match(/^v4-\d+-\d+$/));
     assert(cls3.id !== cls2.id);
 
@@ -512,7 +515,7 @@ describe('onCLS()', async function () {
     assert.strictEqual(cls1.entries.length, 0);
     assert.match(cls1.navigationType, /navigate|reload/);
 
-    assert(cls2.value >= 0);
+    assert(cls2.value > 0);
     assert.strictEqual(cls2.name, 'CLS');
     assert.strictEqual(cls2.id, cls1.id);
     assert.strictEqual(cls2.value, cls1.delta + cls2.delta);
@@ -658,12 +661,12 @@ describe('onCLS()', async function () {
   it('reports if the page is restored from bfcache even when the document was hidden at page load time', async function () {
     if (!browserSupportsCLS) this.skip();
 
-    await navigateTo('/test/cls?hidden=1');
+    await navigateTo('/test/cls?hidden=1', {readyState: 'complete'});
 
     await stubForwardBack();
 
-    // Wait for a frame to be painted.
-    await nextFrame();
+    // clear any beacons from page load.
+    await clearBeacons();
 
     await triggerLayoutShift();
 
@@ -672,7 +675,7 @@ describe('onCLS()', async function () {
 
     const [cls] = await getBeacons();
 
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.delta, cls.value);
@@ -693,7 +696,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
     const [cls] = await getBeacons();
 
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -714,7 +717,7 @@ describe('onCLS()', async function () {
     await beaconCountIs(1);
     const [cls] = await getBeacons();
 
-    assert(cls.value >= 0);
+    assert(cls.value > 0);
     assert(cls.id.match(/^v4-\d+-\d+$/));
     assert.strictEqual(cls.name, 'CLS');
     assert.strictEqual(cls.value, cls.delta);
@@ -736,7 +739,7 @@ describe('onCLS()', async function () {
       await beaconCountIs(1);
 
       const [cls] = await getBeacons();
-      assert(cls.value >= 0);
+      assert(cls.value > 0);
       assert(cls.id.match(/^v4-\d+-\d+$/));
       assert.strictEqual(cls.name, 'CLS');
       assert.strictEqual(cls.value, cls.delta);
@@ -782,7 +785,7 @@ describe('onCLS()', async function () {
       await beaconCountIs(1);
       const [cls] = await getBeacons();
 
-      assert(cls.value >= 0);
+      assert(cls.value > 0);
       assert(cls.id.match(/^v4-\d+-\d+$/));
       assert.strictEqual(cls.name, 'CLS');
       assert.strictEqual(cls.value, cls.delta);
@@ -822,7 +825,7 @@ describe('onCLS()', async function () {
       await beaconCountIs(1);
       const [cls] = await getBeacons();
 
-      assert(cls.value >= 0);
+      assert.strictEqual(cls.value, 0);
       assert(cls.id.match(/^v4-\d+-\d+$/));
       assert.strictEqual(cls.name, 'CLS');
       assert.strictEqual(cls.value, cls.delta);
@@ -838,14 +841,15 @@ describe('onCLS()', async function () {
 let marginTop = 0;
 
 /**
- * Returns a promise that resolves once the browser window has loaded and all
- * the images in the document have decoded and rendered.
- * @return {Promise<void>}
+ * Adds
+ * @return {void}
  */
-function triggerLayoutShift() {
-  return browser.execute((marginTop) => {
+async function triggerLayoutShift() {
+  await browser.execute((marginTop) => {
     document.querySelector('h1').style.marginTop = marginTop + 'em';
   }, ++marginTop);
+  // Wait for a frame to be painted to ensure shifts are finished painting.
+  await nextFrame();
 }
 
 /**