Skip to content

Commit

Permalink
fix(AWS Lambda): Ensure version hash is affected by layer changes
Browse files Browse the repository at this point in the history 
(PR #8066)
  • Loading branch information
@pwithams
pwithams committed Sep 24, 2020
1 parent 76e02cc commit e43c889
Show file tree
Hide file tree
Showing 8 changed files with 422 additions and 144 deletions.
216 changes: 133 additions & 83 deletions lib/plugins/aws/package/compile/functions/index.js
View file
Expand Up @@ -368,6 +368,7 @@ class AwsCompileFunctions {
: this.serverless.service.provider.versionFunctions;

let versionCompilationPromise;

if (shouldVersionFunction || functionObject.provisionedConcurrency) {
// Create hashes for the artifact and the logical id of the version resource
// The one for the version resource must include the function configuration
Expand All @@ -377,105 +378,120 @@ class AwsCompileFunctions {
const versionHash = crypto.createHash('sha256');
fileHash.setEncoding('base64');
versionHash.setEncoding('base64');
const layerConfigurations = _.cloneDeep(
extractLayerConfigurationsFromFunction(functionResource.Properties, cfTemplate)
);

// Read the file in chunks and add them to the hash (saves memory and performance)
versionCompilationPromise = BbPromise.fromCallback(cb => {
const readStream = fs.createReadStream(artifactFilePath);

readStream
.on('data', chunk => {
fileHash.write(chunk);
versionHash.write(chunk);
})
.on('close', () => {
cb();
})
.on('error', error => {
cb(error);
// Include function and layer configuration in version id hash (without the Code part)
versionCompilationPromise = addFileContentsToHashes(artifactFilePath, [
fileHash,
versionHash,
])
.then(async () => {
// Include all referenced layer code in the version id hash
const layerArtifactPaths = [];
layerConfigurations.forEach(layer => {
const layerArtifactPath = this.provider.resolveLayerArtifactName(layer.name);
layerArtifactPaths.push(layerArtifactPath);
});
}).then(() => {
// Include function configuration in version id hash (without the Code part)
const properties = _.omit(
_.get(functionResource, 'Properties', {}),
'Code',
// Properties applied to function globally (not specific to version or alias)
'ReservedConcurrentExecutions',
'Tags'
);
_.forOwn(properties, value => {
if (_.isObject(value)) {
versionHash.write(JSON.stringify(value));
} else {
versionHash.write(value != null ? String(value) : '');

for (const layerArtifactPath of layerArtifactPaths.sort()) {
await addFileContentsToHashes(layerArtifactPath, [versionHash]);
}
})
.then(() => {
// Include function and layer configuration details in the version id hash
for (const layerConfig of layerConfigurations) {
delete layerConfig.properties.Content.S3Key;
}
});

// Finalize hashes
fileHash.end();
versionHash.end();
// sort the layer conifigurations for hash consistency
const sortedLayerConfigurations = {};
const byKey = ([key1], [key2]) => key1.localeCompare(key2);
for (const { name, properties: layerProperties } of layerConfigurations) {
sortedLayerConfigurations[name] = _.fromPairs(
Object.entries(layerProperties).sort(byKey)
);
}

const fileDigest = fileHash.read();
const versionDigest = versionHash.read();
const functionProperties = _.cloneDeep(functionResource.Properties);
delete functionProperties.Code;
// Properties applied to function globally (not specific to version or alias)
delete functionProperties.ReservedConcurrentExecutions;
delete functionProperties.Tags;
functionProperties.layerConfigurations = sortedLayerConfigurations;

const versionResource = this.cfLambdaVersionTemplate();
const sortedFunctionProperties = _.fromPairs(
Object.entries(functionProperties).sort(byKey)
);
versionHash.write(JSON.stringify(sortedFunctionProperties));

versionResource.Properties.CodeSha256 = fileDigest;
versionResource.Properties.FunctionName = { Ref: functionLogicalId };
if (functionObject.description) {
versionResource.Properties.Description = functionObject.description;
}
// Finalize hashes
fileHash.end();
versionHash.end();

// use the version SHA in the logical resource ID of the version because
// AWS::Lambda::Version resource will not support updates
const versionLogicalId = this.provider.naming.getLambdaVersionLogicalId(
functionName,
versionDigest
);
functionObject.versionLogicalId = versionLogicalId;
const newVersionObject = {
[versionLogicalId]: versionResource,
};
const fileDigest = fileHash.read();
const versionDigest = versionHash.read();

Object.assign(cfTemplate.Resources, newVersionObject);
const versionResource = this.cfLambdaVersionTemplate();

// Add function versions to Outputs section
const functionVersionOutputLogicalId = this.provider.naming.getLambdaVersionOutputLogicalId(
functionName
);
const newVersionOutput = this.cfOutputLatestVersionTemplate();
versionResource.Properties.CodeSha256 = fileDigest;
versionResource.Properties.FunctionName = { Ref: functionLogicalId };
if (functionObject.description) {
versionResource.Properties.Description = functionObject.description;
}

newVersionOutput.Value = { Ref: versionLogicalId };
// use the version SHA in the logical resource ID of the version because
// AWS::Lambda::Version resource will not support updates
const versionLogicalId = this.provider.naming.getLambdaVersionLogicalId(
functionName,
versionDigest
);
functionObject.versionLogicalId = versionLogicalId;
const newVersionObject = {
[versionLogicalId]: versionResource,
};

Object.assign(cfTemplate.Outputs, {
[functionVersionOutputLogicalId]: newVersionOutput,
});
Object.assign(cfTemplate.Resources, newVersionObject);

if (!functionObject.provisionedConcurrency) return;
if (!shouldVersionFunction) delete versionResource.DeletionPolicy;
// Add function versions to Outputs section
const functionVersionOutputLogicalId = this.provider.naming.getLambdaVersionOutputLogicalId(
functionName
);
const newVersionOutput = this.cfOutputLatestVersionTemplate();

const provisionedConcurrency = Number(functionObject.provisionedConcurrency);
const aliasLogicalId = this.provider.naming.getLambdaProvisionedConcurrencyAliasLogicalId(
functionName
);
const aliasName = this.provider.naming.getLambdaProvisionedConcurrencyAliasName();

functionObject.targetAlias = { name: aliasName, logicalId: aliasLogicalId };

const aliasResource = {
Type: 'AWS::Lambda::Alias',
Properties: {
FunctionName: { Ref: functionLogicalId },
FunctionVersion: { 'Fn::GetAtt': [versionLogicalId, 'Version'] },
Name: aliasName,
ProvisionedConcurrencyConfig: {
ProvisionedConcurrentExecutions: provisionedConcurrency,
newVersionOutput.Value = { Ref: versionLogicalId };

Object.assign(cfTemplate.Outputs, {
[functionVersionOutputLogicalId]: newVersionOutput,
});

if (!functionObject.provisionedConcurrency) return;
if (!shouldVersionFunction) delete versionResource.DeletionPolicy;

const provisionedConcurrency = Number(functionObject.provisionedConcurrency);
const aliasLogicalId = this.provider.naming.getLambdaProvisionedConcurrencyAliasLogicalId(
functionName
);
const aliasName = this.provider.naming.getLambdaProvisionedConcurrencyAliasName();

functionObject.targetAlias = { name: aliasName, logicalId: aliasLogicalId };

const aliasResource = {
Type: 'AWS::Lambda::Alias',
Properties: {
FunctionName: { Ref: functionLogicalId },
FunctionVersion: { 'Fn::GetAtt': [versionLogicalId, 'Version'] },
Name: aliasName,
ProvisionedConcurrencyConfig: {
ProvisionedConcurrentExecutions: provisionedConcurrency,
},
},
},
DependsOn: functionLogicalId,
};
DependsOn: functionLogicalId,
};

cfTemplate.Resources[aliasLogicalId] = aliasResource;
});
cfTemplate.Resources[aliasLogicalId] = aliasResource;
});
} else {
versionCompilationPromise = BbPromise.resolve();
}
Expand Down Expand Up @@ -629,4 +645,38 @@ class AwsCompileFunctions {
}
}

function addFileContentsToHashes(filePath, hashes) {
return new Promise((resolve, reject) => {
const readStream = fs.createReadStream(filePath);
readStream
.on('data', chunk => {
hashes.forEach(hash => {
hash.write(chunk);
});
})
.on('close', () => {
resolve();
})
.on('error', error => {
reject(new Error(`Could not add file content to hash: ${error}`));
});
});
}

function extractLayerConfigurationsFromFunction(functionProperties, cfTemplate) {
const layerConfigurations = [];
if (!functionProperties.Layers) return layerConfigurations;
functionProperties.Layers.forEach(potentialLocalLayerObject => {
if (potentialLocalLayerObject.Ref) {
const configuration = cfTemplate.Resources[potentialLocalLayerObject.Ref];
layerConfigurations.push({
name: configuration._serverlessLayerName,
ref: potentialLocalLayerObject.Ref,
properties: configuration.Properties,
});
}
});
return layerConfigurations;
}

module.exports = AwsCompileFunctions;

0 comments on commit e43c889

Please sign in to comment.