⬆️ [RUMF-1434] fix yarn failing to install puppeteer on M1 mac (#1843)

* ⬆️ [RUMF-1434] upgrade puppeteer * 🐛 [RUMF-1434] workaround network tracking losing track of requests Sometimes, puppeteer (or the underlying CDP session) don't notify us when a request finishes. This makes the scenario stuck indefinitely. This commit improves the situation by: * listening to the `response` event (it has been observed that sometimes `requestfailed` and `requestfinished` are not notified, but `response` is) * adding a 5 seconds timeout to all requests * 🐛 [RUMF-1434] fix CPU profiling It seems like following the puppeteer upgrade (maybe because it uses a different Chromium version), the Chrome Devtools `Profiler` profile is reset on each page navigation. Thus, we lose some precious information. This commit fix it by stop/start the profiler more often, aggregating the different resulting profiles. * 🐛 [RUMF-1434] fix twitter scenario Twitter does not display the "Preferences" page controls if we don't consent to cookies anymore. This commit adds the expected cookie to opt-in to cookie consent. * 👌 add doc
DataDog · Nov 29, 2022 · eac81e2 · eac81e2
1 parent 76dfb7e
commit eac81e2
Show file tree

Hide file tree

Showing 7 changed files with 108 additions and 85 deletions.
diff --git a/performances/package.json b/performances/package.json
@@ -9,6 +9,6 @@
     "@types/node": "15.0.1",
     "@types/node-forge": "1.0.1",
     "node-forge": "1.3.0",
-    "puppeteer": "8.0.0"
+    "puppeteer": "19.3.0"
   }
 }
diff --git a/performances/src/profilers/startCpuProfiling.ts b/performances/src/profilers/startCpuProfiling.ts
@@ -6,7 +6,10 @@ export async function startCPUProfiling(options: ProfilingOptions, client: CDPSe
   await client.send('Profiler.enable')
   await client.send('Profiler.start')
 
-  return async () => {
+  let totalConsumption = 0
+  let sdkConsumption = 0
+
+  async function stopAndAddProfile() {
     const { profile } = await client.send('Profiler.stop')
 
     const timeDeltaForNodeId = new Map<number, number>()
@@ -16,16 +19,25 @@ export async function startCPUProfiling(options: ProfilingOptions, client: CDPSe
       timeDeltaForNodeId.set(nodeId, (timeDeltaForNodeId.get(nodeId) || 0) + profile.timeDeltas![index])
     }
 
-    let totalConsumption = 0
-    let sdkConsumption = 0
     for (const node of profile.nodes) {
       const consumption = timeDeltaForNodeId.get(node.id) || 0
       totalConsumption += consumption
       if (isSdkBundleUrl(options, node.callFrame.url)) {
         sdkConsumption += consumption
       }
     }
+  }
 
-    return { total: totalConsumption, sdk: sdkConsumption }
+  return {
+    takeCPUMeasurements: async () => {
+      // We need to restart profiling at each "measurement" because the running profile gets reset
+      // on each navigation.
+      await stopAndAddProfile()
+      await client.send('Profiler.start')
+    },
+    stopCPUProfiling: async () => {
+      await stopAndAddProfile()
+      return { total: totalConsumption, sdk: sdkConsumption }
+    },
   }
 }
diff --git a/performances/src/profilers/startProfiling.ts b/performances/src/profilers/startProfiling.ts
@@ -6,12 +6,15 @@ import { startMemoryProfiling } from './startMemoryProfiling'
 
 export async function startProfiling(options: ProfilingOptions, page: Page) {
   const client = await page.target().createCDPSession()
-  const stopCPUProfiling = await startCPUProfiling(options, client)
+  const { stopCPUProfiling, takeCPUMeasurements } = await startCPUProfiling(options, client)
   const { stopMemoryProfiling, takeMemoryMeasurements } = await startMemoryProfiling(options, client)
   const stopNetworkProfiling = await startNetworkProfiling(options, client)
 
   return {
-    takeMeasurements: takeMemoryMeasurements,
+    takeMeasurements: async () => {
+      await takeCPUMeasurements()
+      await takeMemoryMeasurements()
+    },
     stopProfiling: async (): Promise<ProfilingResults> => ({
       memory: await stopMemoryProfiling(),
       cpu: await stopCPUProfiling(),

diff --git a/performances/src/proxy.ts b/performances/src/proxy.ts
@@ -55,6 +55,7 @@ export function startProxy() {
       })
       req.on('end', () => {
         stats.addRequest(req, requestSize)
+        res.setHeader('Access-Control-Allow-Origin', req.headers.origin!)
         res.writeHead(200)
         res.end('{}')
       })

diff --git a/performances/src/scenarios/twitterScenario.ts b/performances/src/scenarios/twitterScenario.ts
@@ -15,6 +15,14 @@ Illustrates a SPA scenario.
 
   async run(page, takeMeasurements) {
     const { waitForNetworkIdle } = trackNetwork(page)
+
+    // Consent to all cookies
+    await page.setCookie({
+      name: 'd_prefs',
+      value: Buffer.from('1:1,consent_version:2,text_version:1000').toString('base64'),
+      domain: 'twitter.com',
+    })
+
     await page.goto('https://twitter.com/explore')
     await waitForNetworkIdle()
 

diff --git a/performances/src/trackNetwork.ts b/performances/src/trackNetwork.ts
@@ -1,36 +1,55 @@
 import type { HTTPRequest, Page } from 'puppeteer'
 
+// Arbitrary maximum time to wait for a request, to make sure `waitForNetworkIdle` does not block
+// for too long.
+const REQUEST_TIMEOUT = 5000 // 5 seconds
+
 export function trackNetwork(page: Page) {
-  const pendingRequests = new Set<HTTPRequest>()
+  const pendingRequests = new Map<HTTPRequest, number>()
 
   page.on('request', (request) => {
-    pendingRequests.add(request)
+    pendingRequests.set(request, Date.now())
   })
   page.on('requestfailed', (request) => {
     pendingRequests.delete(request)
   })
   page.on('requestfinished', (request) => {
     pendingRequests.delete(request)
   })
+  page.on('response', (response) => {
+    pendingRequests.delete(response.request())
+  })
 
   return {
     waitForNetworkIdle: async () =>
       new Promise<void>((resolve) => {
         let timeoutId: NodeJS.Timeout
+        const periodicalWakeIntervalId = setInterval(wake, REQUEST_TIMEOUT)
 
         wake()
 
         page.on('request', wake)
         page.on('requestfinished', wake)
         page.on('requestfailed', wake)
+        page.on('response', wake)
 
         function wake() {
           clearTimeout(timeoutId)
+
+          const now = Date.now()
+          pendingRequests.forEach((start, request) => {
+            if (start < now - REQUEST_TIMEOUT) {
+              pendingRequests.delete(request)
+            }
+          })
+
           if (pendingRequests.size === 0) {
+            clearInterval(periodicalWakeIntervalId)
             timeoutId = setTimeout(() => {
               page.off('request', wake)
               page.off('requestfinished', wake)
               page.off('requestfailed', wake)
+              page.off('response', wake)
               resolve()
             }, 200)
           }

diff --git a/yarn.lock b/yarn.lock
@@ -3158,15 +3158,6 @@ binary-extensions@^2.0.0:
   resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c"
   integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==
 
-bl@^4.0.1:
-  version "4.0.3"
-  resolved "https://registry.yarnpkg.com/bl/-/bl-4.0.3.tgz#12d6287adc29080e22a705e5764b2a9522cdc489"
-  integrity sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==
-  dependencies:
-    buffer "^5.5.0"
-    inherits "^2.0.4"
-    readable-stream "^3.4.0"
-
 bl@^4.0.3, bl@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
@@ -3944,6 +3935,17 @@ cors@2.8.5, cors@~2.8.5:
     object-assign "^4"
     vary "^1"
 
+cosmiconfig@7.0.1:
+  version "7.0.1"
+  resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.1.tgz#714d756522cace867867ccb4474c5d01bbae5d6d"
+  integrity sha512-a1YWNUV2HwGimB7dU2s1wUMurNKjpx60HxBB6xUM8Re+2s1g1IIfJvFR0/iCF+XHdE0GMTKTuLR32UQff4TEyQ==
+  dependencies:
+    "@types/parse-json" "^4.0.0"
+    import-fresh "^3.2.1"
+    parse-json "^5.0.0"
+    path-type "^4.0.0"
+    yaml "^1.10.0"
+
 cosmiconfig@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-7.0.0.tgz#ef9b44d773959cae63ddecd122de23853b60f8d3"
@@ -4092,7 +4094,7 @@ debug@2.6.9, debug@^2.6.8, debug@^2.6.9:
   dependencies:
     ms "2.0.0"
 
-debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
+debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
   integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
@@ -4212,10 +4214,10 @@ detect-indent@^6.0.0:
   resolved "https://registry.yarnpkg.com/detect-indent/-/detect-indent-6.0.0.tgz#0abd0f549f69fc6659a254fe96786186b6f528fd"
   integrity sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==
 
-devtools-protocol@0.0.854822:
-  version "0.0.854822"
-  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.854822.tgz#eac3a5260a6b3b4e729a09fdc0c77b0d322e777b"
-  integrity sha512-xd4D8kHQtB0KtWW0c9xBZD5LVtm9chkMOfs/3Yn01RhT/sFIsVtzTtypfKoFfWBaL+7xCYLxjOLkhwPXaX/Kcg==
+devtools-protocol@0.0.1056733:
+  version "0.0.1056733"
+  resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1056733.tgz#55bb1d56761014cc221131cca5e6bad94eefb2b9"
+  integrity sha512-CmTu6SQx2g3TbZzDCAV58+LTxVdKplS7xip0g5oDXpZ+isr0rv5dDP8ToyVRywzPHkCCPKgKgScEcwz4uPWDIA==
 
 devtools-protocol@0.0.969999:
   version "0.0.969999"
@@ -5062,17 +5064,6 @@ extract-zip@2.0.1:
   optionalDependencies:
     "@types/yauzl" "^2.9.1"
 
-extract-zip@^2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.0.tgz#f53b71d44f4ff5a4527a2259ade000fb8b303492"
-  integrity sha512-i42GQ498yibjdvIhivUsRslx608whtGoFIhF26Z7O4MYncBxp8CwalOs1lnHy21A9sIohWO2+uiE4SRtC9JXDg==
-  dependencies:
-    debug "^4.1.1"
-    get-stream "^5.1.0"
-    yauzl "^2.10.0"
-  optionalDependencies:
-    "@types/yauzl" "^2.9.1"
-
 fast-deep-equal@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz#7b05218ddf9667bf7f370bf7fdb2cb15fdd0aa49"
@@ -5905,6 +5896,14 @@ https-proxy-agent@5.0.0, https-proxy-agent@^5.0.0:
     agent-base "6"
     debug "4"
 
+https-proxy-agent@5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6"
+  integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==
+  dependencies:
+    agent-base "6"
+    debug "4"
+
 https-proxy-agent@^2.2.1:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.3.tgz#fb6cd98ed5b9c35056b5a73cd01a8a721d7193d1"
@@ -8479,7 +8478,7 @@ process-nextick-args@~2.0.0:
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
   integrity sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==
 
-progress@2.0.3, progress@^2.0.1:
+progress@2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/progress/-/progress-2.0.3.tgz#7e8cf8d8f5b8f239c1bc68beb4eb78567d572ef8"
   integrity sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==
@@ -8569,6 +8568,22 @@ punycode@^2.1.0:
   resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.1.1.tgz#b58b010ac40c22c5657616c8d2c2c02c7bf479ec"
   integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==
 
+puppeteer-core@19.3.0:
+  version "19.3.0"
+  resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-19.3.0.tgz#deba854f3dd3f74a04db200274827a67e200f39e"
+  integrity sha512-P8VAAOBnBJo/7DKJnj1b0K9kZBF2D8lkdL94CjJ+DZKCp182LQqYemPI9omUSZkh4bgykzXjZhaVR1qtddTTQg==
+  dependencies:
+    cross-fetch "3.1.5"
+    debug "4.3.4"
+    devtools-protocol "0.0.1056733"
+    extract-zip "2.0.1"
+    https-proxy-agent "5.0.1"
+    proxy-from-env "1.1.0"
+    rimraf "3.0.2"
+    tar-fs "2.1.1"
+    unbzip2-stream "1.4.3"
+    ws "8.10.0"
+
 puppeteer-core@^13.1.3:
   version "13.5.1"
   resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-13.5.1.tgz#74d902d8f4cbc003c4cb15c647bd070cc83539e7"
@@ -8587,23 +8602,17 @@ puppeteer-core@^13.1.3:
     unbzip2-stream "1.4.3"
     ws "8.5.0"
 
-puppeteer@8.0.0:
-  version "8.0.0"
-  resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-8.0.0.tgz#a236669118aa795331c2d0ca19877159e7664705"
-  integrity sha512-D0RzSWlepeWkxPPdK3xhTcefj8rjah1791GE82Pdjsri49sy11ci/JQsAO8K2NRukqvwEtcI+ImP5F4ZiMvtIQ==
+puppeteer@19.3.0:
+  version "19.3.0"
+  resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-19.3.0.tgz#defa8b6a6401b23cdc4f634c441b55d61f6b3d65"
+  integrity sha512-WJbi/ULaeuFOz7cfMgJlJCBAZiyqIFeQ6os4h5ex3PVTt2qosXgwI9eruFZqFAwJRv8x5pOuMhWR0aSRgyDqEg==
   dependencies:
-    debug "^4.1.0"
-    devtools-protocol "0.0.854822"
-    extract-zip "^2.0.0"
-    https-proxy-agent "^5.0.0"
-    node-fetch "^2.6.1"
-    pkg-dir "^4.2.0"
-    progress "^2.0.1"
-    proxy-from-env "^1.1.0"
-    rimraf "^3.0.2"
-    tar-fs "^2.0.0"
-    unbzip2-stream "^1.3.3"
-    ws "^7.2.3"
+    cosmiconfig "7.0.1"
+    devtools-protocol "0.0.1056733"
+    https-proxy-agent "5.0.1"
+    progress "2.0.3"
+    proxy-from-env "1.1.0"
+    puppeteer-core "19.3.0"
 
 pure-color@^1.2.0:
   version "1.3.0"
@@ -9666,16 +9675,6 @@ tar-fs@2.1.1:
     pump "^3.0.0"
     tar-stream "^2.1.4"
 
-tar-fs@^2.0.0:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2"
-  integrity sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==
-  dependencies:
-    chownr "^1.1.1"
-    mkdirp-classic "^0.5.2"
-    pump "^3.0.0"
-    tar-stream "^2.0.0"
-
 tar-stream@2.2.0, tar-stream@^2.1.4, tar-stream@^2.2.0, tar-stream@~2.2.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
@@ -9687,17 +9686,6 @@ tar-stream@2.2.0, tar-stream@^2.1.4, tar-stream@^2.2.0, tar-stream@~2.2.0:
     inherits "^2.0.3"
     readable-stream "^3.1.1"
 
-tar-stream@^2.0.0:
-  version "2.1.2"
-  resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.1.2.tgz#6d5ef1a7e5783a95ff70b69b97455a5968dc1325"
-  integrity sha512-UaF6FoJ32WqALZGOIAApXx+OdxhekNMChu6axLJR85zMMjXKWFGjbIRe+J6P4UnRGg9rAwWvbTT0oI7hD/Un7Q==
-  dependencies:
-    bl "^4.0.1"
-    end-of-stream "^1.4.1"
-    fs-constants "^1.0.0"
-    inherits "^2.0.3"
-    readable-stream "^3.1.1"
-
 tar@^6.0.2, tar@^6.1.0, tar@^6.1.11, tar@^6.1.2:
   version "6.1.11"
   resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621"
@@ -10061,14 +10049,6 @@ unbzip2-stream@1.4.3:
     buffer "^5.2.1"
     through "^2.3.8"
 
-unbzip2-stream@^1.3.3:
-  version "1.4.2"
-  resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.2.tgz#84eb9e783b186d8fb397515fbb656f312f1a7dbf"
-  integrity sha512-pZMVAofMrrHX6Ik39hCk470kulCbmZ2SWfQLPmTWqfJV/oUm0gn1CblvHdUu4+54Je6Jq34x8kY6XjTy6dMkOg==
-  dependencies:
-    buffer "^5.2.1"
-    through "^2.3.8"
-
 unique-filename@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230"
@@ -10606,6 +10586,11 @@ write-pkg@^4.0.0:
     type-fest "^0.4.1"
     write-json-file "^3.2.0"
 
+ws@8.10.0:
+  version "8.10.0"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.10.0.tgz#00a28c09dfb76eae4eb45c3b565f771d6951aa51"
+  integrity sha512-+s49uSmZpvtAsd2h37vIPy1RBusaLawVe8of+GyEPsaJTCMpj/2v8NpeK1SHXjBlQ95lQTmQofOJnFiLoaN3yw==
+
 ws@8.5.0:
   version "8.5.0"
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.5.0.tgz#bfb4be96600757fe5382de12c670dab984a1ed4f"
@@ -10616,11 +10601,6 @@ ws@8.7.0:
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.7.0.tgz#eaf9d874b433aa00c0e0d8752532444875db3957"
   integrity sha512-c2gsP0PRwcLFzUiA8Mkr37/MI7ilIlHQxaEAtd0uNMbVMoy8puJyafRlm0bV9MbGSabUPeLrRRaqIBcFcA2Pqg==
 
-ws@^7.2.3:
-  version "7.4.6"
-  resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
-  integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==
-
 ws@~8.2.3:
   version "8.2.3"
   resolved "https://registry.yarnpkg.com/ws/-/ws-8.2.3.tgz#63a56456db1b04367d0b721a0b80cae6d8becbba"