From 99c12f5de61ac328e9198c83398852c7a4f90628 Mon Sep 17 00:00:00 2001
From: Dzhuneyt <1754428+Dzhuneyt@users.noreply.github.com>
Date: Wed, 1 Jul 2020 15:59:47 +0300
Subject: [PATCH] feat(core): improved docker bundling performance on mac os
 (#8766)

First time contributor to AWS CDK here. Let me know if I've missed something.

As a developer who uses macOS for development purposes on a daily basis and AWS CDK for an enterprise project, I am really impacted by the official and [well known Docker volumes performance issue](https://docs.docker.com/docker-for-mac/osxfs-caching/). Since the project that we are working on, relies heavily on Lambdas that are primarly written in TypeScript, the mass deployment of these Lambdas using AWS CDK CLI and the (recently introduced) NodejsFunction construct takes a significant amount of time and causes a CPU overload.

This is mainly due to the fact that the construct internally does the following things:
1. Starting a docker container with Parcel
2. Mounting the root of the closest folder that contains .git (this is the root of the whole CDK app for us, that includes the CDK stacks, the Lambdas and any other utility functions and node_modules folders, since we opted for the "monorepo" structural pattern)
3. Runs the parcel bundler inside the container, which emits a single .js file as an asset that will serve as the Lambda's "code"
4. Deploys the .js file using the standard lambda.Function construct (NodejsFunction is just a wrapper of lambda.Function with @aws-cdk/core bundling and Parcel involved).

For our project, this means CDK deployment spawns dozens of containers (one per Lambda), that all mount (using Docker volumes), the whole project. The frequent IO operations by Parcel within the container cause a massive CPU spike in all of these containers, causing each Lambda compilation to take 1-2 seconds. This latency is quickly magnified as the number of Lambdas grow within the project.

The latency and CPU spike is mainly a side effect of how Docker and volumes work in macOS for which you can read more on the above link. This is a common pain point for all developers who use these tools, even outside the AWS CDK world. Here are some examples and further reading, including further reading on how flags like ":cached" or ":delegated" help:

- https://github.com/docker/for-mac/issues/1759#issuecomment-590974721
- https://stackoverflow.com/questions/58277794/diagnosing-high-cpu-usage-on-docker-for-mac/58293240#58293240
- https://stackoverflow.com/questions/51694789/macbook-pro-2018-high-temperature-and-cpu-usage-with-docker/51698665#51698665
- https://stackoverflow.com/questions/60878918/docker-hyperkit-process-cpu-usage-going-crazy-how-to-keep-it-under-control#comment107712547_60878918
- https://stackoverflow.com/questions/55951014/docker-in-macos-is-very-slow/55953023#55953023

I've decided to use ":delegated" here since the bundling mechanism generates files inside the container that need replication on the host machine with small tolerable delay, not the other way around (where ":cached" would have been useful). The addition of the flag is non-conditional based on the current OS, because the existence of the flag is a NO-OP in all other operating systems (tested) and is only taken into account by Docker when the volume driver is detected to be osxfs.

Closes https://github.com/aws/aws-cdk/issues/8544
----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
---
 packages/@aws-cdk/core/lib/bundling.ts       | 28 +++++++++++++++++++-
 packages/@aws-cdk/core/test/test.bundling.ts |  2 +-
 packages/@aws-cdk/core/test/test.staging.ts  | 10 +++----
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts
index 1034517534f10..e078153386c6e 100644
--- a/packages/@aws-cdk/core/lib/bundling.ts
+++ b/packages/@aws-cdk/core/lib/bundling.ts
@@ -112,7 +112,7 @@ export class BundlingDockerImage {
       ...options.user
         ? ['-u', options.user]
         : [],
-      ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}`])),
+      ...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}:${v.consistency ?? DockerVolumeConsistency.DELEGATED}`])),
       ...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])),
       ...options.workingDirectory
         ? ['-w', options.workingDirectory]
@@ -138,6 +138,32 @@ export interface DockerVolume {
    * The path where the file or directory is mounted in the container
    */
   readonly containerPath: string;
+
+  /**
+   * Mount consistency. Only applicable for macOS
+   *
+   * @default DockerConsistency.DELEGATED
+   * @see https://docs.docker.com/storage/bind-mounts/#configure-mount-consistency-for-macos
+   */
+  readonly consistency?: DockerVolumeConsistency;
+}
+
+/**
+ * Supported Docker volume consistency types. Only valid on macOS due to the way file storage works on Mac
+ */
+export enum DockerVolumeConsistency {
+  /**
+   * Read/write operations inside the Docker container are applied immediately on the mounted host machine volumes
+   */
+  CONSISTENT = 'consistent',
+  /**
+   * Read/write operations on mounted Docker volumes are first written inside the container and then synchronized to the host machine
+   */
+  DELEGATED = 'delegated',
+  /**
+   * Read/write operations on mounted Docker volumes are first applied on the host machine and then synchronized to the container
+   */
+  CACHED = 'cached',
 }
 
 /**
diff --git a/packages/@aws-cdk/core/test/test.bundling.ts b/packages/@aws-cdk/core/test/test.bundling.ts
index 2ba23a83ffce9..558765010ccfe 100644
--- a/packages/@aws-cdk/core/test/test.bundling.ts
+++ b/packages/@aws-cdk/core/test/test.bundling.ts
@@ -34,7 +34,7 @@ export = {
     test.ok(spawnSyncStub.calledWith('docker', [
       'run', '--rm',
       '-u', 'user:group',
-      '-v', '/host-path:/container-path',
+      '-v', '/host-path:/container-path:delegated',
       '--env', 'VAR1=value1',
       '--env', 'VAR2=value2',
       '-w', '/working-directory',
diff --git a/packages/@aws-cdk/core/test/test.staging.ts b/packages/@aws-cdk/core/test/test.staging.ts
index e229d29ed1e8b..98e8f665de9b7 100644
--- a/packages/@aws-cdk/core/test/test.staging.ts
+++ b/packages/@aws-cdk/core/test/test.staging.ts
@@ -122,7 +122,7 @@ export = {
     const assembly = app.synth();
     test.deepEqual(
       readDockerStubInput(),
-      `run --rm ${USER_ARG} -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS`,
+      `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input alpine DOCKER_STUB_SUCCESS`,
     );
     test.deepEqual(fs.readdirSync(assembly.directory), [
       'asset.2f37f937c51e2c191af66acf9b09f548926008ec68c575bd2ee54b6e997c0e00',
@@ -158,7 +158,7 @@ export = {
 
     test.equal(
       readDockerStubInput(),
-      `run --rm ${USER_ARG} -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS_NO_OUTPUT`,
+      `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input alpine DOCKER_STUB_SUCCESS_NO_OUTPUT`,
     );
     test.done();
   },
@@ -182,7 +182,7 @@ export = {
     // THEN
     test.equal(
       readDockerStubInput(),
-      `run --rm ${USER_ARG} -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS`,
+      `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input alpine DOCKER_STUB_SUCCESS`,
     );
     test.equal(asset.assetHash, '33cbf2cae5432438e0f046bc45ba8c3cef7b6afcf47b59d1c183775c1918fb1f');
 
@@ -226,7 +226,7 @@ export = {
     }), /Cannot specify `bundle` for `assetHashType`/);
     test.equal(
       readDockerStubInput(),
-      `run --rm ${USER_ARG} -v /input:/asset-input -v /output:/asset-output -w /asset-input alpine DOCKER_STUB_SUCCESS`,
+      `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input alpine DOCKER_STUB_SUCCESS`,
     );
 
     test.done();
@@ -280,7 +280,7 @@ export = {
     }), /Failed to run bundling Docker image for asset stack\/Asset/);
     test.equal(
       readDockerStubInput(),
-      `run --rm ${USER_ARG} -v /input:/asset-input -v /output:/asset-output -w /asset-input this-is-an-invalid-docker-image DOCKER_STUB_FAIL`,
+      `run --rm ${USER_ARG} -v /input:/asset-input:delegated -v /output:/asset-output:delegated -w /asset-input this-is-an-invalid-docker-image DOCKER_STUB_FAIL`,
     );
 
     test.done();