From f937f73e6424eac3ce6ab811b72363fc52a27cd9 Mon Sep 17 00:00:00 2001 From: Sebastian Poxhofer Date: Mon, 13 Sep 2021 13:31:01 +0200 Subject: [PATCH] feat(manager/terraform/lockfile): support multiple deps during dep update (#11630) Co-authored-by: Michael Kriese --- .../lockfile/__fixtures__/validLockfile2.hcl | 78 ++++++++++++ .../lockfile/__snapshots__/index.spec.ts.snap | 70 ++++++++++ lib/manager/terraform/lockfile/index.spec.ts | 120 +++++++++++------- lib/manager/terraform/lockfile/index.ts | 63 ++++----- lib/manager/terraform/lockfile/util.ts | 3 + lib/manager/types.ts | 1 + 6 files changed, 256 insertions(+), 79 deletions(-) create mode 100644 lib/manager/terraform/lockfile/__fixtures__/validLockfile2.hcl diff --git a/lib/manager/terraform/lockfile/__fixtures__/validLockfile2.hcl b/lib/manager/terraform/lockfile/__fixtures__/validLockfile2.hcl new file mode 100644 index 00000000000000..fb8589a6ee31ee --- /dev/null +++ b/lib/manager/terraform/lockfile/__fixtures__/validLockfile2.hcl @@ -0,0 +1,78 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.0.0" + constraints = "3.0.0" + hashes = [ + "h1:ULKfwySvQ4pDhy027ryRhLxDhg640wsojYc+7NHMFBU=", + "zh:25294510ae9c250502f2e37ac32b01017439735f098f82a1728772427626a2fd", + "zh:3b723e7772d47bd8cc11bea6e5d3e0b5e1df8398c0e7aaf510e3a8a54e0f1874", + "zh:4b7b73b86f4a0705d5d2a7f1d3ad3279706bdb3957a48f4a389c36918fba838e", + "zh:9e26cdc3be97e3001c253c0ca28c5c8ff2d5476373ca1beb849f3f3957ce7f1a", + "zh:9e73cf1304bf57968d3048d70c0b766d41497430a2a9a7a718a196f3a385106a", + "zh:a30b5b66facfbb2b02814e4cd33ca9899f9ade5bbf478f78c41d2fe789f0582a", + "zh:b06fb5da094db41cb5e430c95c988b73f32695e9f90f25499e926842dbd21b21", + "zh:c5a4ff607e9e9edee3fcd6d6666241fb532adf88ea1fe24f2aa1eb36845b3ca3", + "zh:df568a69087831c1780fac4395630a2cfb3cdf67b7dffbfe16bd78c64770bb75", + "zh:fce1b69dd673aace19508640b0b9b7eb1ef7e746d76cb846b49e7d52e0f5fb7e", + ] +} + +provider "registry.terraform.io/hashicorp/azurerm" { + version = "2.50.0" + constraints = "~> 2.50" + hashes = [ + "h1:Vr6WUm88s9hXGkyVjHtHsP2Jmc2ypQXn6ww7dXtvk1M=", + "zh:0c0688d5a743248f8646d39eb3645a4ac19fd7523ba1b47072fa3fb03b92b1b0", + "zh:2beb3a55ee970f87a9292ae96d57134be8a03d0566117e7be0fe0d9c1267e4ea", + "zh:38091b463fbafe5756420ce34c87845c2a391fec0cded27bdcbbca28febad382", + "zh:4ba455da3b37ba8f8b03ff2781121d9c54d0bd8afd76dfe67593011c475dd73f", + "zh:5d32b9ed871b3c3b774dc69f1fe14cdf7c1fd63d12bb5f21aad4bfbf75e5ee3d", + "zh:6c80cf90a3fc1e17d9caf67cc558c2ff91f8b25e29fdf00942f67711895be5c0", + "zh:c0a53e3165407999d10de7aaa983485d42797433c60b5775791ae299121279ed", + "zh:dab51d6d76041505aeebf20111febe8616ec465ca31dfb7901f5f5c23a5af095", + "zh:e1ad6399f6a6d799002206ee4cb7b794dbb2533b8c3c14502a4419955ec96bff", + "zh:e98f1d178d1e111b3f3449e27d305ce263071226fad3d86272e1bd161c26fd43", + "zh:eb76ec000c9c49a0bf730370c8880f671597bc01f7b7401ab301df7124c049ec", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "2.2.1" + constraints = "~> 2.2" + hashes = [ + "h1:Zg1Bpi6vr7b0H6no8kVDfEucn5pvNALivdrVKVHarGs=", + "zh:072ce92b0138ee65df2e4e2e6e5f6632fa12a7e6453b91399bad89291855d426", + "zh:5731987fe61051515f449033e456ee55207caf17ef41096eb82247810585f53b", + "zh:6f18b10175708bb5839e1f2082dcc02651b876786cd54ec415a091f3821807c3", + "zh:7fa7737661380d18cba3cdc71c4ec6f2fd281b9d61112f6b48d06ca8bbf97771", + "zh:8466cb8fbb4de887b23039082a6e3dc85aeabce86dd808e2a7a65e4e1c51dbae", + "zh:888c63417701c13bbe785ab11dc690d4803e6a2156318cf188970b7b6400b99e", + "zh:a231df55d36fbad1a6705f5d3be4f7459a73ec76117d13f22aa83c10fc610278", + "zh:b62d9a4cd64a2d229070260f4abfef476ebbd7c5511b43e9cdccf23ce938f630", + "zh:b6bd1a325f909bb93f7c9bef00eb306bef1e406cbdf557901d755a3e7a4a5448", + "zh:b9f59afc23cc5567075f76313214baa1e5ce909325229e23c9a4666f7b26e7f7", + "zh:d040220c09b8d9d6bd937572bd5b14bc069af2b883185a873460530d8a1de6e6", + "zh:f254c1f943eb016ae07ebe91b23f813dc79f2064616c65f98c8f64ce23be90c4", + ] +} + +provider "registry.terraform.io/telmate/proxmox" { + version = "2.6.1" + constraints = "~> 2.6.1" + hashes = [ + "h1:eAFb62Hxq4BcKMZXUus2G32/sl0DmLhcBAj6IJeMNo4=", + "zh:0837e6a52120caa538330278c13086f7a7d8c15be2000afdf73fcb2f0d30daa1", + "zh:2964c02fd3eeff4f19aead79c91087e7375eca1bb582036ea1105cd4d5949e2f", + "zh:4540f5fd9db1d2d07466e00a09b610d64ac86ff72ba6f7cbfa8161b07e5c9d04", + "zh:660d6b9b931cc0a2dc8c3c47058448d5cdfcccc38f371441c23e8e5de1a77ba8", + "zh:6e01766d94883a77c1883a71784d6cdc1f04f862fa8087043ce06a4b9d8c9ea6", + "zh:80d8fb293008b9d226996acd158b1a69208b67df15cc15b23a5a24957356400d", + "zh:8cd7def49251517bf65dd8a345ae047dc4dd2e1e6178e4c20e4d473f507b3004", + "zh:a51bd83d57fe718bb5b86d8c464dcd152558ea7bc04bdeb6202690722e5288b5", + "zh:a70f60a5ce57a40857226d8728684bc6a752e1a0003fac0e5cbc87428a87364a", + "zh:b7b27e276c0bb79acb262564db151988d676c96d6384debdf4b7c21bd0967cea", + "zh:c215f5f6a4a34238307294f4900c12c704f99e0e69e9d2a265d40f92b6ccb759", + ] +} diff --git a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap index 182047b7165274..ad42a772025324 100644 --- a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap +++ b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap @@ -137,6 +137,76 @@ Array [ ] `; +exports[`manager/terraform/lockfile/index update multiple dependencies which are not ordered 1`] = ` +Object { + "contents": "# This file is maintained automatically by \\"terraform init\\". +# Manual edits may be lost in future updates. + +provider \\"registry.terraform.io/hashicorp/aws\\" { + version = \\"3.1.0\\" + constraints = \\"~> 3.0\\" + hashes = [ + \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\", + \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\", + ] +} + +provider \\"registry.terraform.io/hashicorp/azurerm\\" { + version = \\"2.56.0\\" + constraints = \\"~> 2.50\\" + hashes = [ + \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\", + \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\", + ] +} + +provider \\"registry.terraform.io/hashicorp/random\\" { + version = \\"3.1.0\\" + constraints = \\"~> 3.0\\" + hashes = [ + \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\", + \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\", + ] +} + +provider \\"registry.terraform.io/telmate/proxmox\\" { + version = \\"2.7.0\\" + constraints = \\"~> 2.7.0\\" + hashes = [ + \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\", + \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\", + ] +} +", + "name": "test/.terraform.lock.hcl", +} +`; + +exports[`manager/terraform/lockfile/index update multiple dependencies which are not ordered 2`] = ` +Array [ + Array [ + "https://registry.terraform.io", + "hashicorp/aws", + "3.1.0", + ], + Array [ + "https://registry.terraform.io", + "hashicorp/random", + "3.1.0", + ], + Array [ + "https://registry.terraform.io", + "hashicorp/azurerm", + "2.56.0", + ], + Array [ + "https://registry.terraform.io", + "telmate/proxmox", + "2.7.0", + ], +] +`; + exports[`manager/terraform/lockfile/index update single dependency in subfolder 1`] = ` Object { "contents": "# This file is maintained automatically by \\"terraform init\\". diff --git a/lib/manager/terraform/lockfile/index.spec.ts b/lib/manager/terraform/lockfile/index.spec.ts index 731d287b5aef07..2964b081bd0fce 100644 --- a/lib/manager/terraform/lockfile/index.spec.ts +++ b/lib/manager/terraform/lockfile/index.spec.ts @@ -22,6 +22,7 @@ const adminConfig = { }; const validLockfile = loadFixture('validLockfile.hcl'); +const validLockfile2 = loadFixture('validLockfile2.hcl'); const mockHash = mocked(TerraformProviderHash).createHashes; const mockGetPkgReleases = getPkgReleases as jest.MockedFunction< @@ -70,13 +71,6 @@ describe('manager/terraform/lockfile/index', () => { 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', ]); - const localConfig: UpdateArtifactsConfig = { - updateType: 'minor', - newVersion: '3.36.0', - newValue: '3.36.0', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'main.tf', updatedDeps: [ @@ -84,10 +78,12 @@ describe('manager/terraform/lockfile/index', () => { depName: 'hashicorp/aws', lookupName: 'hashicorp/aws', depType: 'provider', + newVersion: '3.36.0', + newValue: '3.36.0', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).not.toBeNull(); expect(result).toBeArrayOfSize(1); @@ -107,13 +103,6 @@ describe('manager/terraform/lockfile/index', () => { 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', ]); - const localConfig: UpdateArtifactsConfig = { - updateType: 'minor', - newVersion: '3.36.0', - newValue: '3.36.0', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'main.tf', updatedDeps: [ @@ -121,10 +110,12 @@ describe('manager/terraform/lockfile/index', () => { depName: 'hashicorp/aws', lookupName: 'hashicorp/aws', depType: 'required_provider', + newVersion: '3.36.0', + newValue: '3.36.0', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).not.toBeNull(); expect(result).toBeArrayOfSize(1); @@ -136,13 +127,6 @@ describe('manager/terraform/lockfile/index', () => { }); it('do not update dependency with depType module', async () => { - const localConfig: UpdateArtifactsConfig = { - updateType: 'minor', - newVersion: '3.36.0', - newValue: '3.36.0', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'main.tf', updatedDeps: [ @@ -150,10 +134,12 @@ describe('manager/terraform/lockfile/index', () => { depName: 'terraform-aws-modules/vpc/aws', lookupName: 'terraform-aws-modules/vpc/aws', depType: 'module', + newVersion: '3.36.0', + newValue: '3.36.0', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).toBeNull(); }); @@ -167,13 +153,6 @@ describe('manager/terraform/lockfile/index', () => { 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', ]); - const localConfig: UpdateArtifactsConfig = { - updateType: 'minor', - newVersion: '2.56.0', - newValue: '~> 2.50', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'main.tf', updatedDeps: [ @@ -182,10 +161,12 @@ describe('manager/terraform/lockfile/index', () => { depType: 'provider', lookupName: 'azurerm', registryUrls: ['https://registry.example.com'], + newVersion: '2.56.0', + newValue: '~> 2.50', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).not.toBeNull(); expect(result).toBeArrayOfSize(1); @@ -205,13 +186,6 @@ describe('manager/terraform/lockfile/index', () => { 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', ]); - const localConfig: UpdateArtifactsConfig = { - updateType: 'major', - newVersion: '3.1.0', - newValue: '~> 3.0', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'main.tf', updatedDeps: [ @@ -219,10 +193,12 @@ describe('manager/terraform/lockfile/index', () => { depName: 'random', lookupName: 'hashicorp/random', depType: 'provider', + newVersion: '3.1.0', + newValue: '~> 3.0', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).not.toBeNull(); expect(result).toBeArrayOfSize(1); @@ -242,13 +218,6 @@ describe('manager/terraform/lockfile/index', () => { 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', ]); - const localConfig: UpdateArtifactsConfig = { - updateType: 'major', - newVersion: '3.1.0', - newValue: '~> 3.0', - ...config, - }; - const result = await updateArtifacts({ packageFileName: 'test/main.tf', updatedDeps: [ @@ -256,10 +225,12 @@ describe('manager/terraform/lockfile/index', () => { depName: 'random', lookupName: 'hashicorp/random', depType: 'provider', + newVersion: '3.1.0', + newValue: '~> 3.0', }, ], newPackageFileContent: '', - config: localConfig, + config, }); expect(result).not.toBeNull(); expect(result).toBeArrayOfSize(1); @@ -270,6 +241,59 @@ describe('manager/terraform/lockfile/index', () => { expect(mockHash.mock.calls).toMatchSnapshot(); }); + it('update multiple dependencies which are not ordered', async () => { + fs.readLocalFile.mockResolvedValue(validLockfile2 as any); + fs.getSiblingFileName.mockReturnValue('test/.terraform.lock.hcl'); + + mockHash.mockResolvedValue([ + 'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=', + 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', + ]); + + const result = await updateArtifacts({ + packageFileName: 'test/main.tf', + updatedDeps: [ + { + depName: 'aws', + lookupName: 'hashicorp/aws', + depType: 'provider', + newVersion: '3.1.0', + newValue: '~> 3.0', + }, + { + depName: 'random', + lookupName: 'hashicorp/random', + depType: 'provider', + newVersion: '3.1.0', + newValue: '~> 3.0', + }, + { + depName: 'azurerm', + lookupName: 'hashicorp/azurerm', + depType: 'provider', + newVersion: '2.56.0', + newValue: '~> 2.50', + }, + { + depName: 'proxmox', + lookupName: 'Telmate/proxmox', + depType: 'provider', + newVersion: '2.7.0', + newValue: '~> 2.7.0', + }, + ], + newPackageFileContent: '', + config, + }); + expect(result).not.toBeNull(); + expect(result).toBeArrayOfSize(1); + expect(result[0].file).not.toBeNull(); + expect(result[0].file).toMatchSnapshot(); + + expect(mockHash.mock.calls).toBeArrayOfSize(4); + expect(mockHash.mock.calls).toMatchSnapshot(); + }); + it('do full lock file maintenance', async () => { fs.readLocalFile.mockResolvedValueOnce(validLockfile as any); fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl'); diff --git a/lib/manager/terraform/lockfile/index.ts b/lib/manager/terraform/lockfile/index.ts index 1171d4d4dd1c37..6322f3e6fedc05 100644 --- a/lib/manager/terraform/lockfile/index.ts +++ b/lib/manager/terraform/lockfile/index.ts @@ -80,40 +80,41 @@ export async function updateArtifacts({ // update all locks in the file during maintenance --> only update version in constraints const maintenanceUpdates = await updateAllLocks(locks); updates.push(...maintenanceUpdates); - } else if ( - ['provider', 'required_provider'].includes(updatedDeps[0].depType) - ) { - // update only specific locks but with constrain updates - const dep = updatedDeps[0]; - - const lookupName = dep.lookupName ?? dep.depName; + } else { + const providerDeps = updatedDeps.filter((dep) => + ['provider', 'required_provider'].includes(dep.depType) + ); + for (const dep of providerDeps) { + const lookupName = dep.lookupName ?? dep.depName; - // handle cases like `Telmate/proxmox` - const massagedLookupName = lookupName.toLowerCase(); + // handle cases like `Telmate/proxmox` + const massagedLookupName = lookupName.toLowerCase(); - const repository = massagedLookupName.includes('/') - ? massagedLookupName - : `hashicorp/${massagedLookupName}`; - const registryUrl = dep.registryUrls - ? dep.registryUrls[0] - : TerraformProviderDatasource.defaultRegistryUrls[0]; - const newConstraint = isPinnedVersion(config.newValue) - ? config.newVersion - : config.newValue; - const updateLock = locks.find((value) => value.lookupName === repository); - const update: ProviderLockUpdate = { - newVersion: config.newVersion, - newConstraint, - newHashes: await TerraformProviderHash.createHashes( - registryUrl, - repository, - config.newVersion - ), - ...updateLock, - }; - updates.push(update); + const repository = massagedLookupName.includes('/') + ? massagedLookupName + : `hashicorp/${massagedLookupName}`; + const registryUrl = dep.registryUrls + ? dep.registryUrls[0] + : TerraformProviderDatasource.defaultRegistryUrls[0]; + const newConstraint = isPinnedVersion(dep.newValue) + ? dep.newVersion + : dep.newValue; + const updateLock = locks.find( + (value) => value.lookupName === repository + ); + const update: ProviderLockUpdate = { + newVersion: dep.newVersion, + newConstraint, + newHashes: await TerraformProviderHash.createHashes( + registryUrl, + repository, + dep.newVersion + ), + ...updateLock, + }; + updates.push(update); + } } - // if no updates have been found or there are failed hashes abort if ( updates.length === 0 || diff --git a/lib/manager/terraform/lockfile/util.ts b/lib/manager/terraform/lockfile/util.ts index 47039cf6969c7d..d8282800fa959c 100644 --- a/lib/manager/terraform/lockfile/util.ts +++ b/lib/manager/terraform/lockfile/util.ts @@ -133,6 +133,9 @@ export function writeLockUpdates( const lines = oldLockFileContent.split('\n'); const sections: string[][] = []; + + // sort updates in order of appearance in the lockfile + updates.sort((a, b) => a.lineNumbers.block.start - b.lineNumbers.block.start); updates.forEach((update, index, array) => { // re add leading whitespace let startWhitespace; diff --git a/lib/manager/types.ts b/lib/manager/types.ts index 657a8d853017a3..69358a103eb96a 100644 --- a/lib/manager/types.ts +++ b/lib/manager/types.ts @@ -136,6 +136,7 @@ export interface LookupUpdate { } export interface PackageDependency> extends Package { + newValue?: string; warnings?: ValidationMessage[]; commitMessageTopic?: string; currentDigestShort?: string;