fix(cli): large context causes E2BIG error during synthesis on Linux (#21373)

Linux systems don't support environment variables larger than 128KiB. This change splits the context into two if it's larger than that and stores the overflow into a temporary file. The application then re-constructs the original context from these two sources.

A special case is when this new version of the CLI is used to synthesize an application that depends on an old version of the framework. The application will still consume part of the context, but the CLI warns the user that some of it has been lost.

Since the tree manipulation logic is basically the same as the one used for displaying notices, it was extracted into its own file.

Re-roll #21230
Fixes #19261

----

### All Submissions:

* [x] Have you followed the guidelines in our [Contributing guide?](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md)

### Adding new Unconventional Dependencies:

* [ ] This PR adds new unconventional dependencies following the process described [here](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md/#adding-new-unconventional-dependencies)

### New Features

* [ ] Have you added the new feature to an [integration test](https://github.com/aws/aws-cdk/blob/main/INTEGRATION_TESTS.md)?
	* [ ] Did you use `yarn integ` to deploy the infrastructure and generate the snapshot (i.e. `yarn integ` without `--dry-run`)?

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
Instead of passing the context in an environment variable, the CLI now writes the context to a temporary file and sets an environment variable only with the location. The app then uses that location to read from the file.

Also tested manually on a Linux machine.

Re-roll #21230
Fixes #19261

Author: otaviomacedo

packages/@aws-cdk/core/lib/app.ts

@@ -1,5 +1,6 @@
 import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
+import * as fs from 'fs-extra';
 import { addCustomSynthesis, ICustomSynthesis } from './private/synthesis';
 import { TreeMetadata } from './private/tree-metadata';
 import { Stage } from './stage';
@@ -173,13 +174,13 @@ export class App extends Stage {
       this.node.setContext(k, v);
     }
 
-    // read from environment
-    const contextJson = process.env[cxapi.CONTEXT_ENV];
-    const contextFromEnvironment = contextJson
-      ? JSON.parse(contextJson)
-      : { };
+    // reconstructing the context from the two possible sources:
+    const context = {
+      ...this.readContextFromEnvironment(),
+      ...this.readContextFromTempFile(),
+    };
 
-    for (const [k, v] of Object.entries(contextFromEnvironment)) {
+    for (const [k, v] of Object.entries(context)) {
       this.node.setContext(k, v);
     }
 
@@ -188,6 +189,16 @@ export class App extends Stage {
       this.node.setContext(k, v);
     }
   }
+
+  private readContextFromTempFile() {
+    const location = process.env[cxapi.CONTEXT_OVERFLOW_LOCATION_ENV];
+    return location ? fs.readJSONSync(location) : {};
+  }
+
+  private readContextFromEnvironment() {
+    const contextJson = process.env[cxapi.CONTEXT_ENV];
+    return contextJson ? JSON.parse(contextJson) : {};
+  }
 }
 
 /**

packages/@aws-cdk/core/test/app.test.ts

@@ -1,6 +1,9 @@
+import * as os from 'os';
+import * as path from 'path';
 import { ContextProvider } from '@aws-cdk/cloud-assembly-schema';
 import * as cxapi from '@aws-cdk/cx-api';
 import { Construct } from 'constructs';
+import * as fs from 'fs-extra';
 import { CfnResource, DefaultStackSynthesizer, Stack, StackProps } from '../lib';
 import { Annotations } from '../lib/annotations';
 import { App, AppProps } from '../lib/app';
@@ -101,29 +104,55 @@ describe('app', () => {
     });
   });
 
-  test('context can be passed through CDK_CONTEXT', () => {
-    process.env[cxapi.CONTEXT_ENV] = JSON.stringify({
+  test('context can be passed through CONTEXT_OVERFLOW_LOCATION_ENV', async () => {
+    const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context'));
+    const overflow = path.join(contextDir, 'overflow.json');
+    fs.writeJSONSync(overflow, {
       key1: 'val1',
       key2: 'val2',
     });
+    process.env[cxapi.CONTEXT_OVERFLOW_LOCATION_ENV] = overflow;
+
     const prog = new App();
     expect(prog.node.tryGetContext('key1')).toEqual('val1');
     expect(prog.node.tryGetContext('key2')).toEqual('val2');
   });
 
-  test('context passed through CDK_CONTEXT has precedence', () => {
+  test('context can be passed through CDK_CONTEXT', async () => {
     process.env[cxapi.CONTEXT_ENV] = JSON.stringify({
       key1: 'val1',
       key2: 'val2',
     });
+
+    const prog = new App();
+    expect(prog.node.tryGetContext('key1')).toEqual('val1');
+    expect(prog.node.tryGetContext('key2')).toEqual('val2');
+  });
+
+  test('context passed through CONTEXT_OVERFLOW_LOCATION_ENV is merged with the context passed through CONTEXT_ENV', async () => {
+    const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context'));
+    const contextLocation = path.join(contextDir, 'context-temp.json');
+    fs.writeJSONSync(contextLocation, {
+      key1: 'val1',
+      key2: 'val2',
+    });
+    process.env[cxapi.CONTEXT_OVERFLOW_LOCATION_ENV] = contextLocation;
+
+    process.env[cxapi.CONTEXT_ENV] = JSON.stringify({
+      key3: 'val3',
+      key4: 'val4',
+    });
+
     const prog = new App({
       context: {
-        key1: 'val3',
-        key2: 'val4',
+        key1: 'val5',
+        key2: 'val6',
       },
     });
     expect(prog.node.tryGetContext('key1')).toEqual('val1');
     expect(prog.node.tryGetContext('key2')).toEqual('val2');
+    expect(prog.node.tryGetContext('key3')).toEqual('val3');
+    expect(prog.node.tryGetContext('key4')).toEqual('val4');
   });
 
   test('context passed through finalContext prop has precedence', () => {

packages/@aws-cdk/cx-api/lib/cxapi.ts

@@ -4,6 +4,11 @@
 export const OUTDIR_ENV = 'CDK_OUTDIR';
 export const CONTEXT_ENV = 'CDK_CONTEXT_JSON';
 
+/**
+ * The name of the temporary file where the context is stored.
+ */
+export const CONTEXT_OVERFLOW_LOCATION_ENV = 'CONTEXT_OVERFLOW_LOCATION_ENV';
+
 /**
  * Environment variable set by the CDK CLI with the default AWS account ID.
  */

packages/aws-cdk/lib/api/cxapp/exec.ts

@@ -1,10 +1,14 @@
 import * as childProcess from 'child_process';
+import * as os from 'os';
 import * as path from 'path';
 import * as cxschema from '@aws-cdk/cloud-assembly-schema';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as fs from 'fs-extra';
-import { debug } from '../../logging';
+import * as semver from 'semver';
+import { debug, warning } from '../../logging';
 import { Configuration, PROJECT_CONFIG, USER_DEFAULTS } from '../../settings';
+import { loadTree, some } from '../../tree';
+import { splitBySize } from '../../util/objects';
 import { versionNumber } from '../../version';
 import { SdkProvider } from '../aws-auth';
 
@@ -44,7 +48,6 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom
   context[cxapi.BUNDLING_STACKS] = bundlingStacks;
 
   debug('context:', context);
-  env[cxapi.CONTEXT_ENV] = JSON.stringify(context);
 
   const build = config.settings.get(['build']);
   if (build) {
@@ -83,9 +86,28 @@ export async function execProgram(aws: SdkProvider, config: Configuration): Prom
 
   debug('env:', env);
 
+  const envVariableSizeLimit = os.platform() === 'win32' ? 32760 : 131072;
+  const [smallContext, overflow] = splitBySize(context, spaceAvailableForContext(env, envVariableSizeLimit));
+
+  // Store the safe part in the environment variable
+  env[cxapi.CONTEXT_ENV] = JSON.stringify(smallContext);
+
+  // If there was any overflow, write it to a temporary file
+  let contextOverflowLocation;
+  if (Object.keys(overflow ?? {}).length > 0) {
+    const contextDir = await fs.mkdtemp(path.join(os.tmpdir(), 'cdk-context'));
+    contextOverflowLocation = path.join(contextDir, 'context-overflow.json');
+    fs.writeJSONSync(contextOverflowLocation, overflow);
+    env[cxapi.CONTEXT_OVERFLOW_LOCATION_ENV] = contextOverflowLocation;
+  }
+
   await exec(commandLine.join(' '));
 
-  return createAssembly(outdir);
+  const assembly = createAssembly(outdir);
+
+  contextOverflowCleanup(contextOverflowLocation, assembly);
+
+  return assembly;
 
   function createAssembly(appDir: string) {
     try {
@@ -215,3 +237,33 @@ async function guessExecutable(commandLine: string[]) {
   }
   return commandLine;
 }
+
+function contextOverflowCleanup(location: string | undefined, assembly: cxapi.CloudAssembly) {
+  if (location) {
+    fs.removeSync(path.dirname(location));
+
+    const tree = loadTree(assembly);
+    const frameworkDoesNotSupportContextOverflow = some(tree, node => {
+      const fqn = node.constructInfo?.fqn;
+      const version = node.constructInfo?.version;
+      return (fqn === 'aws-cdk-lib.App' && version != null && semver.lte(version, '2.38.0'))
+        || fqn === '@aws-cdk/core.App'; // v1
+    });
+
+    // We're dealing with an old version of the framework here. It is unaware of the temporary
+    // file, which means that it will ignore the context overflow.
+    if (frameworkDoesNotSupportContextOverflow) {
+      warning('Part of the context could not be sent to the application. Please update the AWS CDK library to the latest version.');
+    }
+  }
+}
+
+function spaceAvailableForContext(env: { [key: string]: string }, limit: number) {
+  const size = (value: string) => value != null ? Buffer.byteLength(value) : 0;
+
+  const usedSpace = Object.entries(env)
+    .map(([k, v]) => k === cxapi.CONTEXT_ENV ? size(k) : size(k) + size(v))
+    .reduce((a, b) => a + b, 0);
+
+  return Math.max(0, limit - usedSpace);
+}

packages/aws-cdk/lib/commands/doctor.ts

@@ -51,7 +51,7 @@ function displayCdkEnvironmentVariables() {
   print('ℹ️ CDK environment variables:');
   let healthy = true;
   for (const key of keys.sort()) {
-    if (key === cxapi.CONTEXT_ENV || key === cxapi.OUTDIR_ENV) {
+    if (key === cxapi.CONTEXT_ENV || key === cxapi.CONTEXT_OVERFLOW_LOCATION_ENV || key === cxapi.OUTDIR_ENV) {
       print(`  - ${chalk.red(key)} = ${chalk.green(process.env[key]!)} (⚠️ reserved for use by the CDK toolkit)`);
       healthy = false;
     } else {

packages/aws-cdk/lib/notices.ts

@@ -3,7 +3,8 @@ import * as https from 'https';
 import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as semver from 'semver';
-import { debug, print, trace } from './logging';
+import { debug, print } from './logging';
+import { some, ConstructTreeNode, loadTreeFromDir } from './tree';
 import { flatMap } from './util';
 import { cdkCacheDir } from './util/directories';
 import { versionNumber } from './version';
@@ -79,7 +80,7 @@ export function filterNotices(data: Notice[], options: FilterNoticeOptions): Not
   const filter = new NoticeFilter({
     cliVersion: options.cliVersion ?? versionNumber(),
     acknowledgedIssueNumbers: options.acknowledgedIssueNumbers ?? new Set(),
-    tree: loadTree(options.outdir ?? 'cdk.out').tree,
+    tree: loadTreeFromDir(options.outdir ?? 'cdk.out'),
   });
   return data.filter(notice => filter.apply(notice));
 }
@@ -336,51 +337,3 @@ function match(query: Component[], tree: ConstructTreeNode): boolean {
     return semver.satisfies(target ?? '', pattern);
   }
 }
-
-function loadTree(outdir: string) {
-  try {
-    return fs.readJSONSync(path.join(outdir, 'tree.json'));
-  } catch (e) {
-    trace(`Failed to get tree.json file: ${e}. Proceeding with empty tree.`);
-    return {};
-  }
-}
-
-/**
- * Source information on a construct (class fqn and version)
- */
-interface ConstructInfo {
-  readonly fqn: string;
-  readonly version: string;
-}
-
-/**
- * A node in the construct tree.
- * @internal
- */
-interface ConstructTreeNode {
-  readonly id: string;
-  readonly path: string;
-  readonly children?: { [key: string]: ConstructTreeNode };
-  readonly attributes?: { [key: string]: any };
-
-  /**
-   * Information on the construct class that led to this node, if available
-   */
-  readonly constructInfo?: ConstructInfo;
-}
-
-function some(node: ConstructTreeNode, predicate: (n: ConstructTreeNode) => boolean): boolean {
-  return node != null && (predicate(node) || findInChildren());
-
-  function findInChildren(): boolean {
-    if (node.children == null) { return false; }
-
-    for (const name in node.children) {
-      if (some(node.children[name], predicate)) {
-        return true;
-      }
-    }
-    return false;
-  }
-}

packages/aws-cdk/lib/tree.ts

@@ -0,0 +1,58 @@
+import * as path from 'path';
+import { CloudAssembly } from '@aws-cdk/cx-api';
+import * as fs from 'fs-extra';
+import { trace } from './logging';
+
+/**
+ * Source information on a construct (class fqn and version)
+ */
+export interface ConstructInfo {
+  readonly fqn: string;
+  readonly version: string;
+}
+
+/**
+ * A node in the construct tree.
+ */
+export interface ConstructTreeNode {
+  readonly id: string;
+  readonly path: string;
+  readonly children?: { [key: string]: ConstructTreeNode };
+  readonly attributes?: { [key: string]: any };
+
+  /**
+   * Information on the construct class that led to this node, if available
+   */
+  readonly constructInfo?: ConstructInfo;
+}
+
+/**
+ * Whether the provided predicate is true for at least one element in the construct (sub-)tree.
+ */
+export function some(node: ConstructTreeNode, predicate: (n: ConstructTreeNode) => boolean): boolean {
+  return node != null && (predicate(node) || findInChildren());
+
+  function findInChildren(): boolean {
+    return Object.values(node.children ?? {}).some(child => some(child, predicate));
+  }
+}
+
+export function loadTree(assembly: CloudAssembly) {
+  try {
+    const outdir = assembly.directory;
+    const fileName = assembly.tree()?.file;
+    return fileName ? fs.readJSONSync(path.join(outdir, fileName)).tree : {};
+  } catch (e) {
+    trace(`Failed to get tree.json file: ${e}. Proceeding with empty tree.`);
+    return {};
+  }
+}
+
+export function loadTreeFromDir(outdir: string) {
+  try {
+    return fs.readJSONSync(path.join(outdir, 'tree.json')).tree;
+  } catch (e) {
+    trace(`Failed to get tree.json file: ${e}. Proceeding with empty tree.`);
+    return {};
+  }
+}

packages/aws-cdk/lib/util/objects.ts

@@ -134,3 +134,39 @@ export function deepMerge(...objects: Array<Obj<any> | undefined>) {
   others.forEach(other => mergeOne(into, other));
   return into;
 }
+
+/**
+ * Splits the given object into two, such that:
+ *
+ * 1. The size of the first object (after stringified in UTF-8) is less than or equal to the provided size limit.
+ * 2. Merging the two objects results in the original one.
+ */
+export function splitBySize(data: any, maxSizeBytes: number): [any, any] {
+  if (maxSizeBytes < 2) {
+    // It's impossible to fit anything in the first object
+    return [undefined, data];
+  }
+  const entries = Object.entries(data);
+  return recurse(0, 0);
+
+  function recurse(index: number, runningTotalSize: number): [any, any] {
+    if (index >= entries.length) {
+      // Everything fits in the first object
+      return [data, undefined];
+    }
+
+    const size = runningTotalSize + entrySize(entries[index]);
+    return (size > maxSizeBytes) ? cutAt(index) : recurse(index + 1, size);
+  }
+
+  function entrySize(entry: [string, unknown]) {
+    return Buffer.byteLength(JSON.stringify(Object.fromEntries([entry])));
+  }
+
+  function cutAt(index: number): [any, any] {
+    return [
+      Object.fromEntries(entries.slice(0, index)),
+      Object.fromEntries(entries.slice(index)),
+    ];
+  }
+}

packages/aws-cdk/package.json

@@ -88,7 +88,8 @@
     "ts-jest": "^27.1.5",
     "ts-mock-imports": "^1.3.8",
     "xml-js": "^1.6.11",
-    "axios": "^0.27.2"
+    "axios": "^0.27.2",
+    "fast-check": "^2.25.0"
   },
   "dependencies": {
     "@aws-cdk/cloud-assembly-schema": "0.0.0",

packages/aws-cdk/test/tree.test.ts

@@ -0,0 +1,114 @@
+import * as path from 'path';
+import { ConstructTreeNode, loadTreeFromDir, some } from '../lib/tree';
+
+describe('some', () => {
+  const tree: ConstructTreeNode = {
+    id: 'App',
+    path: '',
+    children: {
+      Tree: {
+        id: 'Tree',
+        path: 'Tree',
+        constructInfo: {
+          fqn: '@aws-cdk/core.Construct',
+          version: '1.162.0',
+        },
+      },
+      stack: {
+        id: 'stack',
+        path: 'stack',
+        children: {
+          bucket: {
+            id: 'bucket',
+            path: 'stack/bucket',
+            children: {
+              Resource: {
+                id: 'Resource',
+                path: 'stack/bucket/Resource',
+                attributes: {
+                  'aws:cdk:cloudformation:type': 'AWS::S3::Bucket',
+                  'aws:cdk:cloudformation:props': {},
+                },
+                constructInfo: {
+                  fqn: '@aws-cdk/aws-s3.CfnBucket',
+                  version: '1.162.0',
+                },
+              },
+            },
+            constructInfo: {
+              fqn: '@aws-cdk/aws-s3.Bucket',
+              version: '1.162.0',
+            },
+          },
+          CDKMetadata: {
+            id: 'CDKMetadata',
+            path: 'stack/CDKMetadata',
+            children: {
+              Default: {
+                id: 'Default',
+                path: 'stack/CDKMetadata/Default',
+                constructInfo: {
+                  fqn: '@aws-cdk/core.CfnResource',
+                  version: '1.162.0',
+                },
+              },
+              Condition: {
+                id: 'Condition',
+                path: 'stack/CDKMetadata/Condition',
+                constructInfo: {
+                  fqn: '@aws-cdk/core.CfnCondition',
+                  version: '1.162.0',
+                },
+              },
+            },
+            constructInfo: {
+              fqn: '@aws-cdk/core.Construct',
+              version: '1.162.0',
+            },
+          },
+        },
+        constructInfo: {
+          fqn: '@aws-cdk/core.Stack',
+          version: '1.162.0',
+        },
+      },
+    },
+    constructInfo: {
+      fqn: '@aws-cdk/core.App',
+      version: '1.162.0',
+    },
+  };
+
+  test('tree matches predicate', () => {
+    expect(some(tree, node => node.constructInfo?.fqn === '@aws-cdk/aws-s3.Bucket')).toBe(true);
+  });
+
+  test('tree does not match predicate', () => {
+    expect(some(tree, node => node.constructInfo?.fqn === '@aws-cdk/aws-lambda.Function')).toBe(false);
+  });
+
+  test('childless tree', () => {
+    const childless = {
+      id: 'App',
+      path: '',
+      constructInfo: {
+        fqn: '@aws-cdk/core.App',
+        version: '1.162.0',
+      },
+    };
+
+    expect(some(childless, node => node.path.length > 0)).toBe(false);
+  });
+});
+
+describe('loadTreeFromDir', () => {
+  test('can find tree', () => {
+    const tree = loadTreeFromDir(path.join(__dirname, 'cloud-assembly-trees/built-with-1_144_0'));
+    expect(tree.id).toEqual('App');
+  });
+
+  test('cannot find tree', () => {
+    const tree = loadTreeFromDir(path.join(__dirname, 'cloud-assembly-trees/foo'));
+    expect(tree).toEqual({});
+  });
+});

packages/aws-cdk/test/util/objects.test.ts

@@ -1,4 +1,5 @@
-import { deepClone, deepGet, deepMerge, deepSet } from '../../lib/util';
+import * as fc from 'fast-check';
+import { deepClone, deepGet, deepMerge, deepSet, splitBySize } from '../../lib/util';
 
 test('deepSet can set deeply', () => {
   const obj = {};
@@ -44,3 +45,20 @@ test('deepMerge does not overwrite if rightmost is "undefined"', () => {
 
   expect(original).toEqual({ a: 1 });
 });
+
+describe('splitBySize', () => {
+  test('objects are split at the right place', () => {
+    fc.assert(
+      fc.property(fc.object(), fc.integer({ min: 2 }), (data, size) => {
+        const [first, second] = splitBySize(data, size);
+
+        expect(Buffer.from(JSON.stringify(first)).length).toBeLessThanOrEqual(size);
+        expect(merge(first, second)).toEqual(data);
+      }),
+    );
+
+    function merge(fst: any, snd: any) {
+      return { ...(fst ?? {}), ...(snd ?? {}) };
+    }
+  });
+});