feat: merge and save the updated timings file (#130)

* start comparing timings

* merge timings

README.md

@@ -262,7 +262,7 @@ $ npx cypress run --env split=3,splitFile=timings.json
 
 For specs not in the timings file, it will use average duration of the known specs. The timings file might not exist, in this case the specs are split by name. At the end of the run, the duration of all run specs is printed and can be saved into the timings JSON file. **Note:** you would need to combine the timings from different runners into a single JSON file yourself.
 
-If the timings file does not exist yet, the timings will be written into the file after the run finishes.
+If the timings file does not exist yet, the timings will be written into the file after the run finishes. If the file exists, and the new timings have new entries or the existing entries are off by more than 10% duration, the merged file is written back.
 
 ## CI summary
 

src/index.js

@@ -6,7 +6,11 @@ const { getSpecs } = require('find-cypress-specs')
 const ghCore = require('@actions/core')
 const cTable = require('console.table')
 const { getChunk } = require('./chunk')
-const { splitByDuration } = require('./timings')
+const {
+  splitByDuration,
+  hasTimeDifferences,
+  mergeTimings,
+} = require('./timings')
 const { getEnvironmentFlag } = require('./utils')
 const path = require('path')
 const os = require('os')
@@ -281,11 +285,35 @@ function cypressSplit(on, config) {
         }
 
         const timingsString = JSON.stringify(timings, null, 2)
+        console.log(timingsString)
+
         if (!fs.existsSync(SPLIT_FILE)) {
           console.log('%s writing out timings file %s', label, SPLIT_FILE)
           fs.writeFileSync(SPLIT_FILE, timingsString + '\n', 'utf8')
+        } else {
+          const splitFile = JSON.parse(fs.readFileSync(SPLIT_FILE, 'utf8'))
+          const hasUpdatedTimings = hasTimeDifferences(splitFile, timings, 0.1)
+          if (hasUpdatedTimings) {
+            // TODO: merge split file with new timings
+            // do not forget specs not present in the current run!
+            const mergedTimings = mergeTimings(splitFile, timings)
+            const mergedText = JSON.stringify(mergedTimings, null, 2)
+            console.log(
+              '%s writing out updated timings file %s',
+              label,
+              SPLIT_FILE,
+            )
+            debug('previous timings has %d entries', splitFile.durations.length)
+            debug('current timings has %d entries', timings.durations.length)
+            debug(
+              'merged timings has %d entries',
+              mergedTimings.durations.length,
+            )
+            fs.writeFileSync(SPLIT_FILE, mergedText + '\n', 'utf8')
+          } else {
+            console.log('%s spec timings unchanged', label)
+          }
         }
-        console.log(timingsString)
       }
 
       const shouldWriteSummary = getEnvironmentFlag('SPLIT_SUMMARY', true)

src/timings.js

@@ -26,4 +26,64 @@ function splitByDuration(n, list) {
   return { chunks: result, sums }
 }
 
-module.exports = { splitByDuration }
+/**
+ * Compares the original timings file contents with the new spec timings.
+ * Returns true if there are differences and the new timings should be saved.
+ * @param {object} originalTimingsJson JSON loaded from the timings file
+ * @param {object} newTimingsJson new spec timings
+ * @param {number} maxTimeDifference Max relative time difference for a spec 0 to 1
+ * @returns {boolean} if found differences
+ */
+function hasTimeDifferences(
+  originalTimingsJson,
+  newTimingsJson,
+  maxTimeDifference,
+) {
+  // first check if there are new specs
+  const originalSpecNames = originalTimingsJson.durations.map(
+    (item) => item.spec,
+  )
+  const newSpecNames = newTimingsJson.durations.map((item) => item.spec)
+  if (newSpecNames.some((specName) => !originalSpecNames.includes(specName))) {
+    // found new spec, must update the timings file
+    return true
+  }
+
+  return newTimingsJson.durations.some((item) => {
+    const prev = originalTimingsJson.durations.find(
+      (curr) => curr.spec === item.spec,
+    )
+    if (!prev) {
+      // should not happen, all specs should be there
+      return false
+    }
+    // guard against zero
+    const prevDuration = prev.duration || 1
+    const relativeChange =
+      Math.abs(prev.duration - item.duration) / prevDuration
+    return relativeChange > maxTimeDifference
+  })
+}
+
+/**
+ * Merge previous timings with possible new or changed timings
+ * into a new object to be saved.
+ * @param {object} prevTimings JSON loaded from the timings file
+ * @param {object} currTimings new spec timings
+ * @returns {object} Merged object to be saved to the timings file
+ */
+function mergeTimings(prevTimings, currTimings) {
+  const merged = structuredClone(prevTimings)
+  currTimings.durations.forEach((item) => {
+    const found = merged.durations.find((x) => x.spec === item.spec)
+    if (found) {
+      Object.assign(found, item)
+    } else {
+      merged.durations.push(item)
+    }
+  })
+  merged.durations.sort((a, b) => a.spec.localeCompare(b.spec))
+  return merged
+}
+
+module.exports = { splitByDuration, hasTimeDifferences, mergeTimings }

timings.json

@@ -1,28 +1,20 @@
 {
   "durations": [
-    {
-      "spec": "cypress/e2e/chunks.cy.js",
-      "duration": 300
-    },
-    {
-      "spec": "cypress/e2e/spec-a.cy.js",
-      "duration": 10050
-    },
     {
       "spec": "cypress/e2e/spec-b.cy.js",
-      "duration": 10100
+      "duration": 10068
     },
     {
-      "spec": "cypress/e2e/spec-c.cy.js",
-      "duration": 10060
+      "spec": "cypress/e2e/spec-a.cy.js",
+      "duration": 10071
     },
     {
-      "spec": "cypress/e2e/spec-d.cy.js",
-      "duration": 10070
+      "spec": "cypress/e2e/chunks.cy.js",
+      "duration": 230
     },
     {
       "spec": "cypress/e2e/timings.cy.js",
-      "duration": 100
+      "duration": 102
     }
   ]
 }