Skip to content

Commit

Permalink
chore(lambda-nodejs): local bundling (aws#9632)
Browse files Browse the repository at this point in the history
Bundle locally if Parcel v2 is installed.

Closes aws#9120
Closes aws#9639
Closes aws#9153 

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
  • Loading branch information
jogold authored and misterjoshua committed Aug 19, 2020
1 parent 84fde44 commit 10c054d
Show file tree
Hide file tree
Showing 10 changed files with 4,427 additions and 220 deletions.
25 changes: 23 additions & 2 deletions packages/@aws-cdk/aws-lambda-nodejs/README.md
Expand Up @@ -101,5 +101,26 @@ new lambda.NodejsFunction(this, 'my-handler', {

The modules listed in `nodeModules` must be present in the `package.json`'s dependencies. The
same version will be used for installation. If a lock file is detected (`package-lock.json` or
`yarn.lock`) it will be used along with the right installer (`npm` or `yarn`). The modules are
installed in a [Lambda compatible Docker container](https://hub.docker.com/r/amazon/aws-sam-cli-build-image-nodejs12.x).
`yarn.lock`) it will be used along with the right installer (`npm` or `yarn`).

### Local bundling
If Parcel v2 is available it will be used to bundle your code in your environment. Otherwise,
bundling will happen in a [Lambda compatible Docker container](https://hub.docker.com/r/amazon/aws-sam-cli-build-image-nodejs12.x).

For macOS the recommendend approach is to install Parcel as Docker volume performance is really poor.

Parcel v2 can be installed with:

```bash
$ npm install --save-dev parcel@next
```

OR

```bash
$ yarn add --dev @parcel@next
```

To force bundling in a Docker container, set the `forceDockerBundling` prop to `true`. This
is useful if your function relies on node modules that should be installed (`nodeModules` prop, see [above](#install-modules)) in a Lambda compatible environment. This is usually the
case with modules using native dependencies.
169 changes: 169 additions & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/bundlers.ts
@@ -0,0 +1,169 @@
import { spawnSync } from 'child_process';
import * as os from 'os';
import * as path from 'path';
import { Runtime } from '@aws-cdk/aws-lambda';
import * as cdk from '@aws-cdk/core';
import { exec } from './util';

interface BundlerProps {
relativeEntryPath: string;
cacheDir?: string;
environment?: { [key: string]: string };
dependencies?: { [key: string]: string };
installer: Installer;
lockFile?: LockFile;
}

interface LocalBundlerProps extends BundlerProps {
projectRoot: string;
}

/**
* Local Parcel bundler
*/
export class LocalBundler implements cdk.ILocalBundling {
public static get runsLocally(): boolean {
if (LocalBundler._runsLocally !== undefined) {
return LocalBundler._runsLocally;
}
if (os.platform() === 'win32') { // TODO: add Windows support
return false;
}
try {
const parcel = spawnSync(require.resolve('parcel'), ['--version']);
LocalBundler._runsLocally = /^2/.test(parcel.stdout.toString().trim()); // Cache result to avoid unnecessary spawns
return LocalBundler._runsLocally;
} catch {
return false;
}
}

private static _runsLocally?: boolean;

constructor(private readonly props: LocalBundlerProps) {}

public tryBundle(outputDir: string) {
if (!LocalBundler.runsLocally) {
return false;
}

const localCommand = createBundlingCommand({
projectRoot: this.props.projectRoot,
relativeEntryPath: this.props.relativeEntryPath,
cacheDir: this.props.cacheDir,
outputDir,
dependencies: this.props.dependencies,
installer: this.props.installer,
lockFile: this.props.lockFile,
});

exec('bash', ['-c', localCommand], {
env: { ...process.env, ...this.props.environment ?? {} },
stdio: [ // show output
'ignore', // ignore stdio
process.stderr, // redirect stdout to stderr
'inherit', // inherit stderr
],
});
return true;
}
}

interface DockerBundlerProps extends BundlerProps {
buildImage?: boolean;
buildArgs?: { [key: string]: string };
runtime: Runtime;
parcelVersion?: string;
}

/**
* Docker bundler
*/
export class DockerBundler {
public readonly bundlingOptions: cdk.BundlingOptions;

constructor(props: DockerBundlerProps) {
const image = props.buildImage
? cdk.BundlingDockerImage.fromAsset(path.join(__dirname, '../parcel'), {
buildArgs: {
...props.buildArgs ?? {},
IMAGE: props.runtime.bundlingDockerImage.image,
PARCEL_VERSION: props.parcelVersion ?? '2.0.0-beta.1',
},
})
: cdk.BundlingDockerImage.fromRegistry('dummy'); // Do not build if we don't need to

const command = createBundlingCommand({
projectRoot: cdk.AssetStaging.BUNDLING_INPUT_DIR, // project root is mounted at /asset-input
relativeEntryPath: props.relativeEntryPath,
cacheDir: props.cacheDir,
outputDir: cdk.AssetStaging.BUNDLING_OUTPUT_DIR,
installer: props.installer,
lockFile: props.lockFile,
dependencies: props.dependencies,
});

this.bundlingOptions = {
image,
command: ['bash', '-c', command],
environment: props.environment,
workingDirectory: path.dirname(path.join(cdk.AssetStaging.BUNDLING_INPUT_DIR, props.relativeEntryPath)),
};
}
}

interface BundlingCommandOptions extends LocalBundlerProps {
outputDir: string;
}

/**
* Generates bundling command
*/
function createBundlingCommand(options: BundlingCommandOptions): string {
const entryPath = path.join(options.projectRoot, options.relativeEntryPath);
const distFile = path.basename(options.relativeEntryPath).replace(/\.ts$/, '.js');
const parcelCommand: string = chain([
[
'$(node -p "require.resolve(\'parcel\')")', // Parcel is not globally installed, find its "bin"
'build', entryPath.replace(/\\/g, '/'), // Always use POSIX paths in the container
'--target', 'cdk-lambda',
'--dist-dir', options.outputDir, // Output bundle in outputDir (will have the same name as the entry)
'--no-autoinstall',
'--no-scope-hoist',
...options.cacheDir
? ['--cache-dir', path.join(options.projectRoot, options.cacheDir)]
: [],
].join(' '),
// Always rename dist file to index.js because Lambda doesn't support filenames
// with multiple dots and we can end up with multiple dots when using automatic
// entry lookup
distFile !== 'index.js' ? `mv ${options.outputDir}/${distFile} ${options.outputDir}/index.js` : '',
]);

let depsCommand = '';
if (options.dependencies) {
// create dummy package.json, copy lock file if any and then install
depsCommand = chain([
`echo '${JSON.stringify({ dependencies: options.dependencies })}' > ${options.outputDir}/package.json`,
options.lockFile ? `cp ${options.projectRoot}/${options.lockFile} ${options.outputDir}/${options.lockFile}` : '',
`cd ${options.outputDir}`,
`${options.installer} install`,
]);
}

return chain([parcelCommand, depsCommand]);
}

export enum Installer {
NPM = 'npm',
YARN = 'yarn',
}

export enum LockFile {
NPM = 'package-lock.json',
YARN = 'yarn.lock'
}

function chain(commands: string[]): string {
return commands.filter(c => !!c).join(' && ');
}
134 changes: 55 additions & 79 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts
Expand Up @@ -2,6 +2,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as lambda from '@aws-cdk/aws-lambda';
import * as cdk from '@aws-cdk/core';
import { DockerBundler, Installer, LocalBundler, LockFile } from './bundlers';
import { PackageJsonManager } from './package-json-manager';
import { findUp } from './util';

Expand All @@ -24,11 +25,11 @@ export interface ParcelBaseOptions {
readonly sourceMaps?: boolean;

/**
* The cache directory
* The cache directory (relative to the project root)
*
* Parcel uses a filesystem cache for fast rebuilds.
*
* @default - `.cache` in the root directory
* @default - `.parcel-cache` in the working directory
*/
readonly cacheDir?: string;

Expand Down Expand Up @@ -66,7 +67,7 @@ export interface ParcelBaseOptions {
readonly nodeModules?: string[];

/**
* The version of Parcel to use.
* The version of Parcel to use when running in a Docker container.
*
* @default - 2.0.0-beta.1
*/
Expand All @@ -78,6 +79,16 @@ export interface ParcelBaseOptions {
* @default - no build arguments are passed
*/
readonly buildArgs?: { [key:string] : string };

/**
* Force bundling in a Docker container even if local bundling is
* possible.This is useful if your function relies on node modules
* that should be installed (`nodeModules`) in a Lambda compatible
* environment.
*
* @default false
*/
readonly forceDockerBundling?: boolean;
}

/**
Expand Down Expand Up @@ -108,15 +119,7 @@ export class Bundling {
if (!projectRoot) {
throw new Error('Cannot find project root. Please specify it with `projectRoot`.');
}

// Bundling image derived from runtime bundling image (AWS SAM docker image)
const image = cdk.BundlingDockerImage.fromAsset(path.join(__dirname, '../parcel'), {
buildArgs: {
...options.buildArgs ?? {},
IMAGE: options.runtime.bundlingDockerImage.image,
PARCEL_VERSION: options.parcelVersion ?? '2.0.0-beta.1',
},
});
const relativeEntryPath = path.relative(projectRoot, path.resolve(options.entry));

const packageJsonManager = new PackageJsonManager(path.dirname(options.entry));

Expand All @@ -135,6 +138,18 @@ export class Bundling {
}
}

let installer = Installer.NPM;
let lockFile: LockFile | undefined;
if (dependencies) {
// Use npm unless we have a yarn.lock.
if (fs.existsSync(path.join(projectRoot, LockFile.YARN))) {
installer = Installer.YARN;
lockFile = LockFile.YARN;
} else if (fs.existsSync(path.join(projectRoot, LockFile.NPM))) {
lockFile = LockFile.NPM;
}
}

// Configure target in package.json for Parcel
packageJsonManager.update({
targets: {
Expand All @@ -150,79 +165,44 @@ export class Bundling {
},
});

// Entry file path relative to container path
const containerEntryPath = path.join(cdk.AssetStaging.BUNDLING_INPUT_DIR, path.relative(projectRoot, path.resolve(options.entry)));
const distFile = path.basename(options.entry).replace(/\.ts$/, '.js');
const parcelCommand = chain([
[
'$(node -p "require.resolve(\'parcel\')")', // Parcel is not globally installed, find its "bin"
'build', containerEntryPath.replace(/\\/g, '/'), // Always use POSIX paths in the container
'--target', 'cdk-lambda',
'--dist-dir', cdk.AssetStaging.BUNDLING_OUTPUT_DIR, // Output bundle in /asset-output (will have the same name as the entry)
'--no-autoinstall',
'--no-scope-hoist',
...options.cacheDir
? ['--cache-dir', '/parcel-cache']
: [],
].join(' '),
// Always rename dist file to index.js because Lambda doesn't support filenames
// with multiple dots and we can end up with multiple dots when using automatic
// entry lookup
distFile !== 'index.js' ? `mv ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/${distFile} ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/index.js` : '',
]);

let installer = Installer.NPM;
let lockfile: string | undefined;
let depsCommand = '';

if (dependencies) {
// Create a dummy package.json for dependencies that we need to install
fs.writeFileSync(
path.join(projectRoot, '.package.json'),
JSON.stringify({ dependencies }),
);

// Use npm unless we have a yarn.lock.
if (fs.existsSync(path.join(projectRoot, LockFile.YARN))) {
installer = Installer.YARN;
lockfile = LockFile.YARN;
} else if (fs.existsSync(path.join(projectRoot, LockFile.NPM))) {
lockfile = LockFile.NPM;
}

// Move dummy package.json and lock file then install
depsCommand = chain([
`mv ${cdk.AssetStaging.BUNDLING_INPUT_DIR}/.package.json ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/package.json`,
lockfile ? `cp ${cdk.AssetStaging.BUNDLING_INPUT_DIR}/${lockfile} ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR}/${lockfile}` : '',
`cd ${cdk.AssetStaging.BUNDLING_OUTPUT_DIR} && ${installer} install`,
]);
// Local
let localBundler: cdk.ILocalBundling | undefined;
if (!options.forceDockerBundling) {
localBundler = new LocalBundler({
projectRoot,
relativeEntryPath,
cacheDir: options.cacheDir,
environment: options.parcelEnvironment,
dependencies,
installer,
lockFile,
});
}

// Docker
const dockerBundler = new DockerBundler({
runtime: options.runtime,
relativeEntryPath,
cacheDir: options.cacheDir,
environment: options.parcelEnvironment,
buildImage: !LocalBundler.runsLocally || options.forceDockerBundling,
buildArgs: options.buildArgs,
parcelVersion: options.parcelVersion,
dependencies,
installer,
lockFile,
});

return lambda.Code.fromAsset(projectRoot, {
assetHashType: cdk.AssetHashType.BUNDLE,
bundling: {
image,
command: ['bash', '-c', chain([parcelCommand, depsCommand])],
environment: options.parcelEnvironment,
volumes: options.cacheDir
? [{ containerPath: '/parcel-cache', hostPath: options.cacheDir }]
: [],
workingDirectory: path.dirname(containerEntryPath).replace(/\\/g, '/'), // Always use POSIX paths in the container
local: localBundler,
...dockerBundler.bundlingOptions,
},
});
}
}

enum Installer {
NPM = 'npm',
YARN = 'yarn',
}

enum LockFile {
NPM = 'package-lock.json',
YARN = 'yarn.lock'
}

function runtimeVersion(runtime: lambda.Runtime): string {
const match = runtime.name.match(/nodejs(\d+)/);

Expand All @@ -232,7 +212,3 @@ function runtimeVersion(runtime: lambda.Runtime): string {

return match[1];
}

function chain(commands: string[]): string {
return commands.filter(c => !!c).join(' && ');
}

0 comments on commit 10c054d

Please sign in to comment.