Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

AWS Lambda layer build #3110

Merged
merged 23 commits into from
Dec 18, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
6c214f4
Implement extension for AWS Lambda for automatic integration.
marshall-lee Nov 9, 2020
776dbd2
AWS Lambda packages cached
iker-barriocanal Dec 9, 2020
16fabd8
AWS Lambda build workflow
iker-barriocanal Dec 9, 2020
04552f6
Merged AWS workflow to the main one
iker-barriocanal Dec 9, 2020
d961a50
Serverless package versions updated
iker-barriocanal Dec 10, 2020
cd54159
Deleted publish script
iker-barriocanal Dec 10, 2020
ea2be97
Serverless AWS integration docs minor change
iker-barriocanal Dec 10, 2020
43f533e
Set default value in the parameter
iker-barriocanal Dec 14, 2020
443df76
Deleted dependency `path-exists`
iker-barriocanal Dec 14, 2020
107a533
Merge branch 'master' into serverless/awslambda-automation
iker-barriocanal Dec 14, 2020
af71461
Fixed broken promise
iker-barriocanal Dec 14, 2020
f0a9269
Merge branch 'serverless/awslambda-automation' of github.com:getsentr…
iker-barriocanal Dec 14, 2020
94b33b5
ref(aws-lambda): WIP build script
iker-barriocanal Dec 16, 2020
2f6714f
Merge branch 'master' into serverless/awslambda-automation
HazAT Dec 16, 2020
6f6a961
fix(aws-lambda): Correct API function
iker-barriocanal Dec 16, 2020
2d688d8
Merge branch 'serverless/awslambda-automation' of github.com:getsentr…
iker-barriocanal Dec 16, 2020
c4962bf
ref: Revert yarn.lock
HazAT Dec 16, 2020
37dc118
ci: Update
HazAT Dec 16, 2020
48c786d
ref: Reduce imports
iker-barriocanal Dec 16, 2020
6727c3a
ref: Remove dependency in AWS Lambda build script
iker-barriocanal Dec 17, 2020
ffc7e38
ref: Rename AWS Lambda initializer
iker-barriocanal Dec 17, 2020
49a5e60
doc: Remove optional step
iker-barriocanal Dec 18, 2020
6f6e3e4
Merge branch 'master' into serverless/awslambda-automation
iker-barriocanal Dec 18, 2020
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
20 changes: 14 additions & 6 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
iker-barriocanal marked this conversation as resolved.
Show resolved Hide resolved
key: ${{ github.sha }}
- name: Install
run: yarn install
- name: Build
Expand All @@ -43,7 +44,8 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
key: ${{ github.sha }}
- uses: andresz1/size-limit-action@v1.4.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -64,7 +66,8 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
key: ${{ github.sha }}
- run: yarn install
- name: Run Linter
run: yarn lint
Expand All @@ -84,7 +87,8 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
key: ${{ github.sha }}
- run: yarn install
- name: Unit Tests
run: yarn test
Expand All @@ -105,9 +109,11 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
key: ${{ github.sha }}
- name: Pack
run: yarn pack:changed
- run: yarn install
- name: Archive Artifacts
uses: actions/upload-artifact@v2
with:
Expand All @@ -117,6 +123,7 @@ jobs:
${{ github.workspace }}/packages/integrations/build/**
${{ github.workspace }}/packages/tracing/build/**
${{ github.workspace }}/packages/**/*.tgz
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip

job_browserstack_test:
name: BrowserStack
Expand All @@ -134,7 +141,8 @@ jobs:
${{ github.workspace }}/packages/**/build
${{ github.workspace }}/packages/**/dist
${{ github.workspace }}/packages/**/esm
key: ${{ runner.os }}-${{ github.sha }}
${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
key: ${{ github.sha }}
- run: yarn install
- name: Integration Tests
env:
Expand Down
7 changes: 7 additions & 0 deletions packages/node/src/sdk.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,13 @@ export function init(options: NodeOptions = {}): void {
options.dsn = process.env.SENTRY_DSN;
}

if (options.tracesSampleRate === undefined && process.env.SENTRY_TRACES_SAMPLE_RATE) {
const tracesSampleRate = parseFloat(process.env.SENTRY_TRACES_SAMPLE_RATE);
if (isFinite(tracesSampleRate)) {
options.tracesSampleRate = tracesSampleRate;
}
}

if (options.release === undefined) {
const global = getGlobalObject<Window>();
// Prefer env var over global
Expand Down
1 change: 1 addition & 0 deletions packages/serverless/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
dist-awslambda-layer/
14 changes: 13 additions & 1 deletion packages/serverless/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ exports.handler = Sentry.AWSLambda.wrapHandler((event, context, callback) => {
});
```

If you also want to trace performance of all the incoming requests and also outgoing AWS service requests, just set the `tracesSampleRate` option.
If you also want to trace performance of all the incoming requests and also outgoing AWS service requests, just set the `tracesSampleRate` option.

```javascript
import * as Sentry from '@sentry/serverless';
Expand All @@ -53,6 +53,18 @@ Sentry.AWSLambda.init({
});
```

#### Integrate Sentry using internal extension

Another and much simpler way to integrate Sentry to your AWS Lambda function is to add an official layer.

1. Choose Layers -> Add Layer.
2. Specify an ARN: `arn:aws:lambda:us-west-1:TODO:layer:TODO:VERSION`.
3. Go to Environment variables and add:
- `NODE_OPTIONS`: `-r @sentry/serverless/dist/awslambda-auto`.
- `SENTRY_DSN`: `your dsn`.
- `SENTRY_TRACES_SAMPLE_RATE`: a number between 0 and 1 representing the chance a transaction is sent to Sentry. For more information, see [docs](https://docs.sentry.io/platforms/node/guides/aws-lambda/configuration/options/#tracesSampleRate).


### Google Cloud Functions

To use this SDK, call `Sentry.GCPFunction.init(options)` at the very beginning of your JavaScript file.
Expand Down
6 changes: 5 additions & 1 deletion packages/serverless/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,22 @@
"@types/node": "^14.6.4",
"aws-sdk": "^2.765.0",
"eslint": "7.6.0",
"find-up": "^5.0.0",
"google-gax": "^2.9.0",
"jest": "^24.7.1",
"nock": "^13.0.4",
"npm-packlist": "^2.1.4",
"npm-run-all": "^4.1.2",
"prettier": "1.19.0",
"read-pkg": "^5.2.0",
"rimraf": "^2.6.3",
"typescript": "3.7.5"
},
"scripts": {
"build": "run-p build:es5 build:esm",
"build": "run-p build:es5 build:esm build:awslambda-layer",
"build:es5": "tsc -p tsconfig.build.json",
"build:esm": "tsc -p tsconfig.esm.json",
"build:awslambda-layer": "node scripts/build-awslambda-layer.js",
"build:watch": "run-p build:watch:es5 build:watch:esm",
"build:watch:es5": "tsc -p tsconfig.build.json -w --preserveWatchOutput",
"build:watch:esm": "tsc -p tsconfig.esm.json -w --preserveWatchOutput",
Expand Down
161 changes: 161 additions & 0 deletions packages/serverless/scripts/build-awslambda-layer.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
const path = require('path');
const process = require('process');
const fs = require('fs');
const childProcess = require('child_process');

const findUp = require('find-up');
const packList = require('npm-packlist');
const readPkg = require('read-pkg');

const serverlessPackage = require('../package.json');

// AWS Lambda layer are being uploaded as zip archive, whose content is then being unpacked to the /opt
// directory in the lambda environment.
//
// So this script does the following: it builds a 'dist-awslambda-layer/nodejs/node_modules/@sentry/serverless'
// directory with a special index.js and with all necessary @sentry packages symlinked as node_modules.
// Then, this directory is compressed with zip.
//
// The tricky part about it is that one cannot just symlink the entire package directories into node_modules because
// all the src/ contents and other unnecessary files will end up in the zip archive. So, we need to symlink only
// individual files from package and it must be only those of them that are distributable.
// There exists a `npm-packlist` library for such purpose. So we need to traverse all the dependencies,
// execute `npm-packlist` on them and symlink the files into 'dist-awslambda-layer/.../@sentry/serverless/node_modules'.
// I didn't find any way to achieve this goal using standard command-line tools so I have to write this script.
//
// Another, and much simpler way to assemble such zip bundle is install all the dependencies from npm registry and
// just bundle the entire node_modules.
// It's easier and looks more stable but it's inconvenient if one wants build a zip bundle out of current source tree.
//
// And yet another way is to bundle everything with webpack into a single file. I tried and it seems to be error-prone
// so I think it's better to have a classic package directory with node_modules file structure.

/** Recursively traverse all the dependencies and collect all the info to the map */
async function collectPackages(cwd, packages = {}) {
const packageJson = await readPkg({ cwd });

packages[packageJson.name] = { cwd, packageJson };

if (!packageJson.dependencies) {
return packages;
}

await Promise.all(
Object.keys(packageJson.dependencies).map(async dep => {
// We are interested only in 'external' dependencies which are strictly upper than current directory.
// Internal deps aka local node_modules folder of each package is handled differently.
const searchPath = path.resolve(cwd, '..');
const depPath = fs.realpathSync(
await findUp(path.join('node_modules', dep),
{ type: 'directory', cwd: searchPath })
);
if (packages[dep]) {
if (packages[dep].cwd != depPath) {
throw new Error(`${packageJson.name}'s dependenciy ${dep} maps to both ${packages[dep].cwd} and ${depPath}`);
}
return;
}
await collectPackages(depPath, packages);
}),
);

return packages;
}

async function main() {
const workDir = path.resolve(__dirname, '..'); // packages/serverless directory
const packages = await collectPackages(workDir);

const dist = path.resolve(workDir, 'dist-awslambda-layer');
const destRootRelative = 'nodejs/node_modules/@sentry/serverless';
const destRoot = path.resolve(dist, destRootRelative);
const destModulesRoot = path.resolve(destRoot, 'node_modules');

try {
// Setting `force: true` ignores exceptions when paths don't exist.
fs.rmSync(destRoot, { force: true, recursive: true, maxRetries: 1 });
fs.mkdirSync(destRoot, { recursive: true });
} catch (error) {
// Ignore errors.
}

await Promise.all(
Object.entries(packages).map(async ([name, pkg]) => {
const isRoot = name == serverlessPackage.name;
const destPath = isRoot ? destRoot : path.resolve(destModulesRoot, name);

// Scan over the distributable files of the module and symlink each of them.
const sourceFiles = await packList({ path: pkg.cwd });
await Promise.all(
sourceFiles.map(async filename => {
const sourceFilename = path.resolve(pkg.cwd, filename);
const destFilename = path.resolve(destPath, filename);

try {
fs.mkdirSync(path.dirname(destFilename), { recursive: true });
fs.symlinkSync(sourceFilename, destFilename);
} catch (error) {
// Ignore errors.
}
}),
);

const sourceModulesRoot = path.resolve(pkg.cwd, 'node_modules');
// `fs.constants.F_OK` indicates whether the file is visible to the current process, but it doesn't check
// its permissions. For more information, refer to https://nodejs.org/api/fs.html#fs_file_access_constants.
try {
fs.accessSync(path.resolve(sourceModulesRoot), fs.constants.F_OK);
} catch (error) {
return;
}

// Scan over local node_modules folder of the package and symlink its non-dev dependencies.
const sourceModules = fs.readdirSync(sourceModulesRoot);
await Promise.all(
sourceModules.map(async sourceModule => {
if (!pkg.packageJson.dependencies || !pkg.packageJson.dependencies[sourceModule]) {
return;
}

const sourceModulePath = path.resolve(sourceModulesRoot, sourceModule);
const destModulePath = path.resolve(destPath, 'node_modules', sourceModule);

try {
fs.mkdirSync(path.dirname(destModulePath), { recursive: true });
fs.symlinkSync(sourceModulePath, destModulePath);
} catch (error) {
// Ignore errors.
}
}),
);
}),
);

const version = serverlessPackage.version;
const zipFilename = `sentry-node-serverless-${version}.zip`;

try {
fs.unlinkSync(path.resolve(dist, zipFilename));
} catch (error) {
// If the ZIP file hasn't been previously created (e.g. running this script for the first time),
// `unlinkSync` will try to delete a non-existing file. This error is ignored.
}

try {
childProcess.execSync(`zip -r ${zipFilename} ${destRootRelative}`, { cwd: dist });
} catch (error) {
// The child process timed out or had non-zero exit code.
// The error contains the entire result from `childProcess.spawnSync`.
console.log(error); // eslint-disable-line no-console
}
}

main().then(
() => {
process.exit(0);
},
err => {
console.error(err); // eslint-disable-line no-console
process.exit(-1);
},
);
13 changes: 13 additions & 0 deletions packages/serverless/src/awslambda-auto.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import * as Sentry from './index';

const lambdaTaskRoot = process.env.LAMBDA_TASK_ROOT;
if (lambdaTaskRoot) {
const handlerString = process.env._HANDLER;
if (!handlerString) {
throw Error(`LAMBDA_TASK_ROOT is non-empty(${lambdaTaskRoot}) but _HANDLER is not set`);
}
Sentry.AWSLambda.init();
Sentry.AWSLambda.tryPatchHandler(lambdaTaskRoot, handlerString);
} else {
throw Error('LAMBDA_TASK_ROOT environment variable is not set');
}
59 changes: 59 additions & 0 deletions packages/serverless/src/awslambda.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,13 @@ import {
} from '@sentry/node';
import * as Sentry from '@sentry/node';
import { Integration } from '@sentry/types';
import { logger } from '@sentry/utils';
// NOTE: I have no idea how to fix this right now, and don't want to waste more time, as it builds just fine — Kamil
// eslint-disable-next-line import/no-unresolved
import { Context, Handler } from 'aws-lambda';
import { existsSync } from 'fs';
import { hostname } from 'os';
import { basename, resolve } from 'path';
import { performance } from 'perf_hooks';
import { types } from 'util';

Expand Down Expand Up @@ -57,6 +60,62 @@ export function init(options: Sentry.NodeOptions = {}): void {
Sentry.addGlobalEventProcessor(serverlessEventProcessor('AWSLambda'));
}

/** */
function tryRequire<T>(taskRoot: string, subdir: string, mod: string): T {
const lambdaStylePath = resolve(taskRoot, subdir, mod);
if (existsSync(lambdaStylePath) || existsSync(`${lambdaStylePath}.js`)) {
// Lambda-style path
return require(lambdaStylePath);
}
// Node-style path
return require(require.resolve(mod, { paths: [taskRoot, subdir] }));
}

/** */
export function tryPatchHandler(taskRoot: string, handlerPath: string): void {
type HandlerBag = HandlerModule | Handler | null | undefined;
interface HandlerModule {
[key: string]: HandlerBag;
}

const handlerDesc = basename(handlerPath);
const match = handlerDesc.match(/^([^.]*)\.(.*)$/);
if (!match) {
logger.error(`Bad handler ${handlerDesc}`);
return;
}

const [, handlerMod, handlerName] = match;

let obj: HandlerBag;
try {
const handlerDir = handlerPath.substring(0, handlerPath.indexOf(handlerDesc));
obj = tryRequire(taskRoot, handlerDir, handlerMod);
} catch (e) {
logger.error(`Cannot require ${handlerPath} in ${taskRoot}`, e);
return;
}

let mod: HandlerBag;
let functionName: string | undefined;
handlerName.split('.').forEach(name => {
mod = obj;
obj = obj && (obj as HandlerModule)[name];
functionName = name;
});
iker-barriocanal marked this conversation as resolved.
Show resolved Hide resolved
if (!obj) {
logger.error(`${handlerPath} is undefined or not exported`);
return;
}
if (typeof obj !== 'function') {
logger.error(`${handlerPath} is not a function`);
return;
}

// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
(mod as HandlerModule)[functionName!] = wrapHandler(obj as Handler);
iker-barriocanal marked this conversation as resolved.
Show resolved Hide resolved
}

/**
* Adds additional information from the environment and AWS Context to the Sentry Scope.
*
Expand Down