fix(cli): template created during import should be written to assets …

…folder (#29830) ### Issue # (if applicable) Closes #22928 #22530 I'm reopening as the original was closed due to inactivity. ### Reason for this change When the template is synthesized, if it is larger than 50kb and cannot be inlined in the CFN request, CLI writes it to the filesystem to be uploaded to S3. Unlike regular deployments `import` overrides the template, when it does so the code responsible for writing the large template to the filesystem writes to the project root. This code reproduces the issue: ``` import * as cdk from "aws-cdk-lib"; import { Construct } from "constructs"; import * as sqs from "aws-cdk-lib/aws-sqs"; import * as iam from "aws-cdk-lib/aws-iam"; export class AwsCdkImportBugStack extends cdk.Stack { constructor(scope: Construct, id: string, props?: cdk.StackProps) { super(scope, id, props); for (let i = 0; i < 70; i++) { const q = new sqs.Queue(this, `AwsCdkImportBugQueue${i}`, { visibilityTimeout: cdk.Duration.seconds(300), enforceSSL: true, }); } // new sqs.Queue(this, `AwsCdkImportBugQueue`, { // visibilityTimeout: cdk.Duration.seconds(300), // enforceSSL: true, // removalPolicy: cdk.RemovalPolicy.RETAIN, // }); } } ``` Deploy the stack, uncomment the queue, and try to import. This error happens: ``` $ npx cdk import AwsCdkImportBugStack AwsCdkImportBugStack/AwsCdkImportBugQueue/Resource (AWS::SQS::Queue): enter QueueUrl (empty to skip): https://sqs.eu-central-1.amazonaws.com/<cut>/AwsCdkBugStack-keep186A2ECA-IznVNwytuY1b AwsCdkImportBugStack/AwsCdkImportBugQueue/Policy/Resource (AWS::SQS::QueuePolicy): enter Id (empty to skip): Skipping import of AwsCdkImportBugStack/AwsCdkImportBugQueue/Policy/Resource AwsCdkImportBugStack: importing resources into stack... [0%] start: Publishing 06a2c6415fd2982e3285e9d615e5f9b99d1d795a9064479fbe6b88455ac62f6c:current [100%] fail: ENOENT: no such file or directory, open '/home/nburtsev/projects/aws-cdk-import-bug/cdk.out/AwsCdkImportBugStack.template.json-06a2c6415fd2982e3285e9d615e5f9b99d1d795a9064479fbe6b88455ac62f6c.yaml' ❌ AwsCdkImportBugStack failed: Error: Failed to publish one or more assets. See the error messages above for more information. at publishAssets (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:420:84876) at process.processTicksAndRejections (node:internal/process/task_queues:95:5) at async deployStack (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:430:391) at async ResourceImporter.importResources (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:433:171694) at async ResourceImporter.importResourcesFromMap (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:433:171357) at async CdkToolkit.import (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:433:205058) at async exec4 (/home/nburtsev/projects/aws-cdk-import-bug/node_modules/aws-cdk/lib/index.js:488:54378) Failed to publish one or more assets. See the error messages above for more information. ``` The file is created, it's just in the project root not in cdk.out: ``` ~/projects/aws-cdk-import-bug $ ls -la total 284 drwxr-xr-x 7 nburtsev users 4096 Mar 22 10:48 . drwxr-xr-x 8 nburtsev users 4096 Mar 22 10:11 .. -rw-r--r-- 1 nburtsev users 64131 Mar 22 10:48 AwsCdkImportBugStack.template.json-06a2c6415fd2982e3285e9d615e5f9b99d1d795a9064479fbe6b88455ac62f6c.yaml drwxr-xr-x 2 nburtsev users 4096 Mar 22 10:12 bin -rw-r--r-- 1 nburtsev users 3185 Mar 22 10:12 cdk.json drwxr-xr-x 2 nburtsev users 4096 Mar 22 10:48 cdk.out <cut> ``` ### Description of changes I've just added a path.join similar to how it is done in regular template generation. ### Description of how you validated changes I've added a test case but I believe tests are broken globally as per @TheRealAmazonKendra ### Checklist - [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md) ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
aws · May 10, 2024 · a96cf55 · a96cf55
1 parent eb39d9e
commit a96cf55
Show file tree

Hide file tree

Showing 3 changed files with 71 additions and 19 deletions.
diff --git a/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js b/packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js
@@ -248,10 +248,37 @@ class MigrateStack extends cdk.Stack {
   }
 }
 
-class ImportableStack extends MigrateStack {
+class ImportableStack extends cdk.Stack {
   constructor(parent, id, props) {
     super(parent, id, props);
     new cdk.CfnWaitConditionHandle(this, 'Handle');
+
+    if (process.env.INCLUDE_SINGLE_QUEUE === '1') {
+      const queue = new sqs.Queue(this, 'Queue', {
+        removalPolicy: (process.env.RETAIN_SINGLE_QUEUE === '1') ? cdk.RemovalPolicy.RETAIN : cdk.RemovalPolicy.DESTROY,
+      });
+
+      new cdk.CfnOutput(this, 'QueueName', {
+        value: queue.queueName,
+      });
+
+      new cdk.CfnOutput(this, 'QueueUrl', {
+        value: queue.queueUrl,
+      });
+
+      new cdk.CfnOutput(this, 'QueueLogicalId', {
+        value: queue.node.defaultChild.logicalId,
+      });
+    }
+
+    if (process.env.LARGE_TEMPLATE === '1') {
+      for (let i = 1; i <= 70; i++) {
+        new sqs.Queue(this, `cdk-import-queue-test${i}`, {
+          enforceSSL: true,
+          removalPolicy: cdk.RemovalPolicy.DESTROY,
+        });
+      }
+    }
   }
 }
 

diff --git a/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts b/packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts
@@ -1619,37 +1619,60 @@ integTest('skips notice refresh', withDefaultFixture(async (fixture) => {
 }));
 
 /**
- * Create a queue with a fresh name, redeploy orphaning the queue, then import it again
+ * Create a queue, orphan that queue, then import the queue.
+ *
+ * We want to test with a large template to make sure large templates can work with import.
  */
 integTest('test resource import', withDefaultFixture(async (fixture) => {
-  const outputsFile = path.join(fixture.integTestDir, 'outputs', 'outputs.json');
+  // GIVEN
+  const randomPrefix = randomString();
+  const uniqueOutputsFileName = `${randomPrefix}Outputs.json`; // other tests use the outputs file. Make sure we don't collide.
+  const outputsFile = path.join(fixture.integTestDir, 'outputs', uniqueOutputsFileName);
   await fs.mkdir(path.dirname(outputsFile), { recursive: true });
 
-  // Initial deploy
+  // First, create a stack that includes many queues, and one queue that will be removed from the stack but NOT deleted from AWS.
   await fixture.cdkDeploy('importable-stack', {
-    modEnv: { ORPHAN_TOPIC: '1' },
+    modEnv: { LARGE_TEMPLATE: '1', INCLUDE_SINGLE_QUEUE: '1', RETAIN_SINGLE_QUEUE: '1' },
     options: ['--outputs-file', outputsFile],
   });
 
-  const outputs = JSON.parse((await fs.readFile(outputsFile, { encoding: 'utf-8' })).toString());
-  const queueName = outputs.QueueName;
-  const queueLogicalId = outputs.QueueLogicalId;
-  fixture.log(`Setup complete, created queue ${queueName}`);
   try {
-    // Deploy again, orphaning the queue
+
+    // Second, now the queue we will remove is in the stack and has a logicalId. We can now make the resource mapping file.
+    // This resource mapping file will be used to tell the import operation what queue to bring into the stack.
+    const fullStackName = fixture.fullStackName('importable-stack');
+    const outputs = JSON.parse((await fs.readFile(outputsFile, { encoding: 'utf-8' })).toString());
+    const queueLogicalId = outputs[fullStackName].QueueLogicalId;
+    const queueResourceMap = {
+      [queueLogicalId]: { QueueUrl: outputs[fullStackName].QueueUrl },
+    };
+    const mappingFile = path.join(fixture.integTestDir, 'outputs', `${randomPrefix}Mapping.json`);
+    await fs.writeFile(
+      mappingFile,
+      JSON.stringify(queueResourceMap),
+      { encoding: 'utf-8' },
+    );
+
+    // Third, remove the queue from the stack, but don't delete the queue from AWS.
     await fixture.cdkDeploy('importable-stack', {
-      modEnv: { OMIT_TOPIC: '1' },
+      modEnv: { LARGE_TEMPLATE: '1', INCLUDE_SINGLE_QUEUE: '0', RETAIN_SINGLE_QUEUE: '0' },
     });
+    const cfnTemplateBeforeImport = await fixture.aws.cloudFormation('getTemplate', { StackName: fullStackName });
+    expect(cfnTemplateBeforeImport.TemplateBody).not.toContain(queueLogicalId);
 
-    // Write a resource mapping file based on the ID from step one, then run an import
-    const mappingFile = path.join(fixture.integTestDir, 'outputs', 'mapping.json');
-    await fs.writeFile(mappingFile, JSON.stringify({ [queueLogicalId]: { QueueName: queueName } }), { encoding: 'utf-8' });
+    // WHEN
+    await fixture.cdk(
+      ['import', '--resource-mapping', mappingFile, fixture.fullStackName('importable-stack')],
+      { modEnv: { LARGE_TEMPLATE: '1', INCLUDE_SINGLE_QUEUE: '1', RETAIN_SINGLE_QUEUE: '0' } },
+    );
 
-    await fixture.cdk(['import',
-      '--resource-mapping', mappingFile,
-      fixture.fullStackName('importable-stack')]);
+    // THEN
+    const describeStacksResponse = await fixture.aws.cloudFormation('describeStacks', { StackName: fullStackName });
+    const cfnTemplateAfterImport = await fixture.aws.cloudFormation('getTemplate', { StackName: fullStackName });
+    expect(describeStacksResponse.Stacks![0].StackStatus).toEqual('IMPORT_COMPLETE');
+    expect(cfnTemplateAfterImport.TemplateBody).toContain(queueLogicalId);
   } finally {
-    // Cleanup
+    // Clean up
     await fixture.cdkDestroy('importable-stack');
   }
 }));

diff --git a/packages/aws-cdk/lib/api/util/template-body-parameter.ts b/packages/aws-cdk/lib/api/util/template-body-parameter.ts
@@ -1,3 +1,4 @@
+import * as path from 'path';
 import * as cxapi from '@aws-cdk/cx-api';
 import * as chalk from 'chalk';
 import * as fs from 'fs-extra';
@@ -68,7 +69,8 @@ export async function makeBodyParameter(
   if (overrideTemplate) {
     // Add a variant of this template
     templateFile = `${stack.templateFile}-${templateHash}.yaml`;
-    await fs.writeFile(templateFile, templateJson, { encoding: 'utf-8' });
+    const templateFilePath = path.join(stack.assembly.directory, templateFile);
+    await fs.writeFile(templateFilePath, templateJson, { encoding: 'utf-8' });
   }
 
   assetManifest.addFileAsset(templateHash, {