feat: split specs based on timings from a JSON file (#125)

* start work on splitting specs based on timings

* testing timings

* add readme file

.github/workflows/ci.yml

@@ -129,6 +129,17 @@ jobs:
         with:
           command: npm run user-specs
 
+  test-timings:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout 🛎
+        uses: actions/checkout@v4
+      - name: Split specs based on timings json file 🧪
+        # https://github.com/cypress-io/github-action
+        uses: cypress-io/github-action@v5
+        with:
+          command: npm run timings
+
   test-workflow-e2e:
     # https://github.com/bahmutov/cypress-workflows
     uses: bahmutov/cypress-workflows/.github/workflows/split.yml@v1
@@ -173,6 +184,7 @@ jobs:
         test-user-spec-list,
         test-subfolder,
         test-index1,
+        test-timings,
       ]
     runs-on: ubuntu-22.04
     steps:

README.md

@@ -229,6 +229,39 @@ Some CIs provide an agent index that already starts at 1. You can pass it via `S
 job1: SPLIT=3 SPLIT_INDEX1=1 npx cypress run
 ```
 
+## Split specs based on timings
+
+If you know the spec timings, you can create a JSON file and pass the timings to this plugin. The list of specs will be split into N machines to make the total durations for each machine approximately equal. You can see an example [timings.json](./timings.json) file:
+
+```json
+{
+  "durations": [
+    {
+      "spec": "cypress/e2e/chunks.cy.js",
+      "duration": 300
+    },
+    {
+      "spec": "cypress/e2e/spec-a.cy.js",
+      "duration": 10050
+    },
+    ...
+  ]
+}
+```
+
+You can pass the JSON filename via `SPLIT_FILE` environment variable or Cypress`env` variable.
+
+```
+# split all specs across 3 machines using known spec timings
+# loaded from "timings.json" file
+$ SPLIT_FILE=timings.json SPLIT=3 npx cypress run
+
+# the equivalent syntax using Cypress --env argument
+$ npx cypress run --env split=3,splitFile=timings.json
+```
+
+For specs not in the timings file, it will use average duration of the known specs.
+
 ## CI summary
 
 To skip GitHub Actions summary, set an environment variable `SPLIT_SUMMARY=false`. By default, this plugin generates the summary.

cypress/e2e/timings.cy.js

@@ -0,0 +1,98 @@
+/// <reference types="cypress" />
+
+const { splitByDuration } = require('../../src/timings')
+
+it('splits specs based on timings', () => {
+  const list = [
+    {
+      spec: 'a',
+      duration: 1000,
+    },
+    {
+      spec: 'b',
+      duration: 6000,
+    },
+    {
+      spec: 'c',
+      duration: 6000,
+    },
+    {
+      spec: 'd',
+      duration: 1000,
+    },
+  ]
+  const { chunks, sums } = splitByDuration(2, list)
+  expect(chunks, 'chunks').to.deep.equal([
+    [
+      {
+        spec: 'b',
+        duration: 6000,
+      },
+      {
+        spec: 'a',
+        duration: 1000,
+      },
+    ],
+    [
+      {
+        spec: 'c',
+        duration: 6000,
+      },
+      {
+        spec: 'd',
+        duration: 1000,
+      },
+    ],
+  ])
+  expect(sums, 'duration sums').to.deep.equal([7000, 7000])
+})
+
+it('splits specs based on timings, single result', () => {
+  const list = [
+    {
+      spec: 'a',
+      duration: 1000,
+    },
+    {
+      spec: 'b',
+      duration: 6000,
+    },
+    {
+      spec: 'c',
+      duration: 6000,
+    },
+    {
+      spec: 'd',
+      duration: 1000,
+    },
+  ]
+  const { chunks, sums } = splitByDuration(4, list)
+  expect(chunks).to.deep.equal([
+    [
+      {
+        spec: 'b',
+        duration: 6000,
+      },
+    ],
+    [
+      {
+        spec: 'c',
+        duration: 6000,
+      },
+    ],
+    [
+      {
+        spec: 'a',
+        duration: 1000,
+      },
+    ],
+
+    [
+      {
+        spec: 'd',
+        duration: 1000,
+      },
+    ],
+  ])
+  expect(sums, 'duration sums').to.deep.equal([6000, 6000, 1000, 1000])
+})

package.json

@@ -17,7 +17,8 @@
     "test-names": "find-cypress-specs --names",
     "test-names:component": "find-cypress-specs --component --names",
     "deps": "npm audit --report --omit dev",
-    "subfolder": "DEBUG=cypress-split,find-cypress-specs SPLIT=2 SPLIT_INDEX=0 cypress run --config-file examples/my-app/tests/cypress.config.js"
+    "subfolder": "DEBUG=cypress-split,find-cypress-specs SPLIT=2 SPLIT_INDEX=0 cypress run --config-file examples/my-app/tests/cypress.config.js",
+    "timings": "DEBUG=cypress-split SPLIT=2 SPLIT_INDEX=0 SPLIT_FILE=timings.json cypress run"
   },
   "repository": {
     "type": "git",

src/index.js

@@ -6,6 +6,7 @@ const { getSpecs } = require('find-cypress-specs')
 const ghCore = require('@actions/core')
 const cTable = require('console.table')
 const { getChunk } = require('./chunk')
+const { splitByDuration } = require('./timings')
 const { getEnvironmentFlag } = require('./utils')
 const path = require('path')
 const os = require('os')
@@ -68,6 +69,7 @@ function cypressSplit(on, config) {
 
   let SPLIT = process.env.SPLIT || config.env.split || config.env.SPLIT
   let SPLIT_INDEX = process.env.SPLIT_INDEX || config.env.splitIndex
+  let SPLIT_FILE = process.env.SPLIT_FILE || config.env.splitFile
 
   // some CI systems like TeamCity provide agent index starting with 1
   // let's check for SPLIT_INDEX1 and if it is set,
@@ -84,6 +86,7 @@ function cypressSplit(on, config) {
 
   // potentially a list of files to run / split
   let SPEC = process.env.SPEC || config.env.spec || config.env.SPEC
+  /** @type {string[]|undefined} absolute spec filenames */
   let specs
   if (typeof SPEC === 'string' && SPEC) {
     specs = SPEC.split(',')
@@ -146,12 +149,52 @@ function cypressSplit(on, config) {
 
     debug('get chunk %o', { specs, splitN, splitIndex })
     /** @type {string[]} absolute spec filenames */
-    const splitSpecs = getChunk(specs, splitN, splitIndex)
-    debug('split specs')
-    debug(splitSpecs)
+    let splitSpecs
 
     const cwd = process.cwd()
     console.log('spec from the current directory %s', cwd)
+
+    if (SPLIT_FILE) {
+      debug('loading split file %s', SPLIT_FILE)
+      const splitFile = JSON.parse(fs.readFileSync(SPLIT_FILE, 'utf8'))
+      const previousDurations = splitFile.durations
+      const averageDuration =
+        previousDurations
+          .map((item) => item.duration)
+          .reduce((sum, duration) => (sum += duration), 0) /
+        previousDurations.length
+      const specsWithDurations = specs.map((specName) => {
+        const relativeSpec = path.relative(cwd, specName)
+        const foundInfo = previousDurations.find(
+          (item) => item.spec === relativeSpec,
+        )
+        if (!foundInfo) {
+          return {
+            specName,
+            duration: averageDuration,
+          }
+        } else {
+          return {
+            specName,
+            duration: foundInfo.duration,
+          }
+        }
+      })
+      debug('splitting by duration %d ways', splitN)
+      debug(specsWithDurations)
+      const { chunks, sums } = splitByDuration(splitN, specsWithDurations)
+      debug('split by duration')
+      debug(chunks)
+      debug('sums of durations for chunks')
+      debug(sums)
+
+      splitSpecs = chunks[splitIndex].map((item) => item.specName)
+    } else {
+      splitSpecs = getChunk(specs, splitN, splitIndex)
+    }
+    debug('split specs')
+    debug(splitSpecs)
+
     const nameRows = splitSpecs.map((specName, k) => {
       const row = [String(k + 1), path.relative(cwd, specName)]
       return row

src/timings.js

@@ -0,0 +1,29 @@
+/**
+ * Split the list of items into "n" lists by "duration" property
+ * in each item. Sorts the list first, then round-robin fills
+ * the lists. Put the item into the list with the smallest sum.
+ * @param {number} n Number of output lists
+ * @returns {any}
+ */
+function splitByDuration(n, list) {
+  const result = []
+  const sums = []
+  for (let k = 0; k < n; k += 1) {
+    result.push([])
+    sums.push(0)
+  }
+  const sorted = list.sort((a, b) => b.duration - a.duration)
+  sorted.forEach((item) => {
+    const smallestIndex = sums.reduce((currentSmallestIndex, value, index) => {
+      return value < sums[currentSmallestIndex] ? index : currentSmallestIndex
+    }, 0)
+    result[smallestIndex].push(item)
+    sums[smallestIndex] += item.duration
+  })
+
+  // console.table(result)
+  // console.table(sums)
+  return { chunks: result, sums }
+}
+
+module.exports = { splitByDuration }

timings.json

@@ -0,0 +1,28 @@
+{
+  "durations": [
+    {
+      "spec": "cypress/e2e/chunks.cy.js",
+      "duration": 300
+    },
+    {
+      "spec": "cypress/e2e/spec-a.cy.js",
+      "duration": 10050
+    },
+    {
+      "spec": "cypress/e2e/spec-b.cy.js",
+      "duration": 10100
+    },
+    {
+      "spec": "cypress/e2e/spec-c.cy.js",
+      "duration": 10060
+    },
+    {
+      "spec": "cypress/e2e/spec-d.cy.js",
+      "duration": 10070
+    },
+    {
+      "spec": "cypress/e2e/timings.cy.js",
+      "duration": 100
+    }
+  ]
+}