fix: Resolve logs getting blocked when an excessive payload is encoun… (#2013)

Author: jsumners-nr

lib/collector/api.js

@@ -480,10 +480,31 @@ CollectorAPI.prototype._sendData = function _sendData(method, data, callback) {
   this._runLifecycle(method, data, (error, response) => {
     // Any runtime errors should preserve the agent run.
     if (error) {
-      // TODO: log error
-      // TODO: differentiate between network (retain) and non-network (drop).
+      let retainData = true
+      if (error.code && error.code === 'NR_REMOTE_METHOD_MAX_PAYLOAD_SIZE_EXCEEDED') {
+        // We are going to drop whatever was in `data`.
+        // The collector + aggregator system collects events into batches until
+        // a harvest interval is reached. At that time, all collected events
+        // are popped off the queue into an array. That array is then serialized
+        // (and possibly compressed) before being sent. If that serialized data
+        // exceeds our allowed limit, there isn't any way to reduce items to
+        // send data, and requeue events, until the data meets the allowed
+        // limit. So we are going to drop it all here, and _not_ pass an error
+        // back to the sending routines.
+        //
+        // Ideally, we'd have an algorithm that:
+        // 1. Calculates if the collected events to be sent will exceed the
+        // allowed size
+        // 2. If so, pop events off the to-send array and re-queues them until
+        // the allowed size is met.
+        // 3. If the paired down to-send array reaches one element that will
+        // still fail, drop that item from the queue.
+        // 4. Let the next cycle do the same thing.
+        retainData = false
+        return callback(null, { retainData })
+      }
 
-      callback(error, { retainData: true })
+      callback(error, { retainData })
 
       return
     }

lib/collector/remote-method.js

@@ -245,7 +245,9 @@ RemoteMethod.prototype._safeRequest = function _safeRequest(options) {
       this.name,
       maxPayloadSize
     )
-    throw new Error('Maximum payload size exceeded')
+    const error = new Error('Maximum payload size exceeded')
+    error.code = 'NR_REMOTE_METHOD_MAX_PAYLOAD_SIZE_EXCEEDED'
+    throw error
   }
 
   // If trace level is not explicitly enabled check to see if the audit log is

lib/config/index.js

@@ -29,7 +29,7 @@ const configDefinition = definition()
  * CONSTANTS -- we gotta lotta 'em
  */
 const AZURE_APP_NAME = 'APP_POOL_ID'
-const DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 1000000
+const DEFAULT_MAX_PAYLOAD_SIZE_IN_BYTES = 1_000_000
 const BASE_CONFIG_PATH = require.resolve('../../newrelic')
 const HAS_ARBITRARY_KEYS = new Set(['ignore_messages', 'expected_messages', 'labels'])
 

test/unit/collector/api.test.js

@@ -8,6 +8,7 @@
 const tap = require('tap')
 
 const nock = require('nock')
+const crypto = require('crypto')
 const helper = require('../../lib/agent_helper')
 const CollectorApi = require('../../../lib/collector/api')
 
@@ -74,6 +75,44 @@ tap.test('reportSettings', (t) => {
       t.end()
     })
   })
+
+  t.test('handles excessive payload sizes without blocking subsequent sends', (t) => {
+    const tstamp = 1_707_756_300_000 // 2024-02-12T11:45:00.000-05:00
+    function log(data) {
+      return JSON.stringify({
+        level: 30,
+        time: tstamp,
+        pid: 17035,
+        hostname: 'test-host',
+        msg: data
+      })
+    }
+
+    const kb512 = log(crypto.randomBytes(524_288).toString('base64'))
+    const mb1 = log(crypto.randomBytes(1_048_576).toString('base64'))
+    const toFind = log('find me')
+
+    let sends = 0
+    const ncontext = nock(URL)
+      .post(helper.generateCollectorPath('log_event_data', RUN_ID))
+      .times(2)
+      .reply(200)
+
+    agent.logs.on('finished log_event_data data send.', () => {
+      sends += 1
+      if (sends === 3) {
+        t.equal(ncontext.isDone(), true)
+        t.end()
+      }
+    })
+
+    agent.logs.add(kb512)
+    agent.logs.send()
+    agent.logs.add(mb1)
+    agent.logs.send()
+    agent.logs.add(toFind)
+    agent.logs.send()
+  })
 })
 
 /**