From dc15dfd80851a3d333a851be6607db6e154259bd Mon Sep 17 00:00:00 2001 From: Pete Wagner <1559510+thepwagner@users.noreply.github.com> Date: Fri, 10 Sep 2021 06:54:57 -0400 Subject: [PATCH] feat: kustomize image digests (#11153) Co-authored-by: Michael Kriese Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> Co-authored-by: Rhys Arkins --- .../kustomize/__fixtures__/digest.yaml | 18 ++++ .../kustomize/__fixtures__/newName.yaml | 10 +++ .../kustomize/__fixtures__/newTag.yaml | 11 +++ lib/manager/kustomize/__fixtures__/sha.yaml | 17 ---- .../__snapshots__/extract.spec.ts.snap | 89 +++++++++++++++++-- lib/manager/kustomize/extract.spec.ts | 81 ++++++++++++++--- lib/manager/kustomize/extract.ts | 76 +++++++++++----- lib/manager/kustomize/readme.md | 45 ++++++++-- lib/manager/kustomize/types.ts | 1 + 9 files changed, 286 insertions(+), 62 deletions(-) create mode 100644 lib/manager/kustomize/__fixtures__/digest.yaml create mode 100644 lib/manager/kustomize/__fixtures__/newName.yaml create mode 100644 lib/manager/kustomize/__fixtures__/newTag.yaml delete mode 100644 lib/manager/kustomize/__fixtures__/sha.yaml diff --git a/lib/manager/kustomize/__fixtures__/digest.yaml b/lib/manager/kustomize/__fixtures__/digest.yaml new file mode 100644 index 00000000000000..720db6cf00e25f --- /dev/null +++ b/lib/manager/kustomize/__fixtures__/digest.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hasura +images: + - name: postgres + digest: sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c + - name: postgres:11 + digest: sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c + # invalid - includes newTag and digest + - name: postgres + newTag: 11 + digest: sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c + # invalid - not a string + - name: postgres + digest: 02641143766 + # invalid - missing prefix + - name: postgres + digest: b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c diff --git a/lib/manager/kustomize/__fixtures__/newName.yaml b/lib/manager/kustomize/__fixtures__/newName.yaml new file mode 100644 index 00000000000000..e9101b827fe495 --- /dev/null +++ b/lib/manager/kustomize/__fixtures__/newName.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hasura +images: + - name: postgres + newName: awesome/postgres:11@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c + - name: postgres + newName: awesome/postgres:11 + - name: postgres + newName: awesome/postgres@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c diff --git a/lib/manager/kustomize/__fixtures__/newTag.yaml b/lib/manager/kustomize/__fixtures__/newTag.yaml new file mode 100644 index 00000000000000..2a7199b1e572ed --- /dev/null +++ b/lib/manager/kustomize/__fixtures__/newTag.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: hasura +images: + - name: postgres + newTag: "11" + - name: postgres + newTag: 11@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c + # invalid - renders as `postgres:sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c` + - name: postgres + newTag: sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c diff --git a/lib/manager/kustomize/__fixtures__/sha.yaml b/lib/manager/kustomize/__fixtures__/sha.yaml deleted file mode 100644 index 9afc12f4284406..00000000000000 --- a/lib/manager/kustomize/__fixtures__/sha.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -namespace: hasura - -commonLabels: - app.kubernetes.io/name: hasura - -bases: - - ../base/ - -patchesStrategicMerge: - - patches/deployment.yaml - -images: - - name: postgres - newTag: sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c diff --git a/lib/manager/kustomize/__snapshots__/extract.spec.ts.snap b/lib/manager/kustomize/__snapshots__/extract.spec.ts.snap index 55eaee54613589..f1b7067ab3ac6f 100644 --- a/lib/manager/kustomize/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/kustomize/__snapshots__/extract.spec.ts.snap @@ -1,5 +1,68 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`manager/kustomize/extract extractPackageFile() extracts from digest 1`] = ` +Object { + "deps": Array [ + Object { + "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "currentValue": undefined, + "datasource": "docker", + "depName": "postgres", + "replaceString": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + }, + Object { + "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "currentValue": "11", + "datasource": "docker", + "depName": "postgres", + "replaceString": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + }, + Object { + "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "currentValue": 11, + "depName": "postgres", + "skipReason": "invalid-dependency-specification", + }, + Object { + "currentValue": 2641143766, + "depName": "postgres", + "skipReason": "invalid-value", + }, + Object { + "currentValue": "b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "depName": "postgres", + "skipReason": "invalid-value", + }, + ], +} +`; + +exports[`manager/kustomize/extract extractPackageFile() extracts from newTag 1`] = ` +Object { + "deps": Array [ + Object { + "currentDigest": undefined, + "currentValue": "11", + "datasource": "docker", + "depName": "postgres", + "replaceString": "11", + }, + Object { + "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "currentValue": "11", + "datasource": "docker", + "depName": "postgres", + "replaceString": "11@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + }, + Object { + "currentValue": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "depName": "postgres", + "skipReason": "invalid-value", + }, + ], +} +`; + exports[`manager/kustomize/extract extractPackageFile() extracts http dependency 1`] = ` Array [ Object { @@ -32,16 +95,29 @@ Array [ ] `; -exports[`manager/kustomize/extract extractPackageFile() extracts sha256 instead of tag 1`] = ` +exports[`manager/kustomize/extract extractPackageFile() extracts newName 1`] = ` Object { "deps": Array [ + Object { + "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + "currentValue": "11", + "datasource": "docker", + "depName": "awesome/postgres", + "replaceString": "awesome/postgres:11@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", + }, + Object { + "currentDigest": undefined, + "currentValue": "11", + "datasource": "docker", + "depName": "awesome/postgres", + "replaceString": "awesome/postgres:11", + }, Object { "currentDigest": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", "currentValue": undefined, "datasource": "docker", - "depName": "postgres", - "replaceString": "sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", - "versioning": "docker", + "depName": "awesome/postgres", + "replaceString": "awesome/postgres@sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c", }, ], } @@ -95,7 +171,6 @@ Array [ "datasource": "docker", "depName": "node", "replaceString": "v0.1.0", - "versioning": "docker", }, Object { "currentDigest": undefined, @@ -103,7 +178,6 @@ Array [ "datasource": "docker", "depName": "group/instance", "replaceString": "v0.0.1", - "versioning": "docker", }, Object { "currentDigest": undefined, @@ -111,7 +185,6 @@ Array [ "datasource": "docker", "depName": "quay.io/test/repo", "replaceString": "v0.0.2", - "versioning": "docker", }, Object { "currentDigest": undefined, @@ -119,7 +192,6 @@ Array [ "datasource": "docker", "depName": "gitlab.com/org/suborg/image", "replaceString": "v0.0.3", - "versioning": "docker", }, Object { "currentDigest": undefined, @@ -127,7 +199,6 @@ Array [ "datasource": "docker", "depName": "but.this.lives.on.local/private-registry", "replaceString": "v0.0.4", - "versioning": "docker", }, Object { "currentValue": 2.5, diff --git a/lib/manager/kustomize/extract.spec.ts b/lib/manager/kustomize/extract.spec.ts index 44c2b71ca9c099..8121a8920ebf1e 100644 --- a/lib/manager/kustomize/extract.spec.ts +++ b/lib/manager/kustomize/extract.spec.ts @@ -3,7 +3,6 @@ import * as datasourceDocker from '../../datasource/docker'; import * as datasourceGitTags from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { SkipReason } from '../../types'; -import * as dockerVersioning from '../../versioning/docker'; import { extractBase, extractImage, @@ -19,7 +18,9 @@ const kustomizeWithLocal = loadFixture('kustomizeWithLocal.yaml'); const nonKustomize = loadFixture('service.yaml'); const gitImages = loadFixture('gitImages.yaml'); const kustomizeDepsInResources = loadFixture('depsInResources.yaml'); -const sha = loadFixture('sha.yaml'); +const newTag = loadFixture('newTag.yaml'); +const newName = loadFixture('newName.yaml'); +const digest = loadFixture('digest.yaml'); describe('manager/kustomize/extract', () => { it('should successfully parse a valid kustomize file', () => { @@ -131,7 +132,6 @@ describe('manager/kustomize/extract', () => { currentValue: 'v1.0.0', datasource: datasourceDocker.id, replaceString: 'v1.0.0', - versioning: dockerVersioning.id, depName: 'node', }; const pkg = extractImage({ @@ -146,7 +146,6 @@ describe('manager/kustomize/extract', () => { currentValue: 'v1.0.0', datasource: datasourceDocker.id, replaceString: 'v1.0.0', - versioning: dockerVersioning.id, depName: 'test/node', }; const pkg = extractImage({ @@ -161,7 +160,6 @@ describe('manager/kustomize/extract', () => { currentValue: 'v1.0.0', datasource: datasourceDocker.id, replaceString: 'v1.0.0', - versioning: dockerVersioning.id, depName: 'quay.io/repo/image', }; const pkg = extractImage({ @@ -175,7 +173,6 @@ describe('manager/kustomize/extract', () => { currentDigest: undefined, currentValue: 'v1.0.0', datasource: datasourceDocker.id, - versioning: dockerVersioning.id, replaceString: 'v1.0.0', depName: 'localhost:5000/repo/image', }; @@ -191,7 +188,6 @@ describe('manager/kustomize/extract', () => { currentValue: 'v1.0.0', replaceString: 'v1.0.0', datasource: datasourceDocker.id, - versioning: dockerVersioning.id, depName: 'localhost:5000/repo/image/service', }; const pkg = extractImage({ @@ -253,13 +249,76 @@ describe('manager/kustomize/extract', () => { expect(res.deps[1].depName).toEqual('fluxcd/flux'); expect(res.deps[2].depName).toEqual('fluxcd/flux'); }); - it('extracts sha256 instead of tag', () => { - expect(extractPackageFile(sha)).toMatchSnapshot({ + + const postgresDigest = + 'sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c'; + + it('extracts from newTag', () => { + expect(extractPackageFile(newTag)).toMatchSnapshot({ deps: [ { - currentDigest: - 'sha256:b0cfe264cb1143c7c660ddfd5c482464997d62d6bc9f97f8fdf3deefce881a8c', + currentDigest: undefined, + currentValue: '11', + replaceString: '11', + }, + { + currentDigest: postgresDigest, + currentValue: '11', + replaceString: `11@${postgresDigest}`, + }, + { + skipReason: SkipReason.InvalidValue, + }, + ], + }); + }); + + it('extracts from digest', () => { + expect(extractPackageFile(digest)).toMatchSnapshot({ + deps: [ + { + currentDigest: postgresDigest, + currentValue: undefined, + replaceString: postgresDigest, + }, + { + currentDigest: postgresDigest, + currentValue: '11', + replaceString: postgresDigest, + }, + { + skipReason: SkipReason.InvalidDependencySpecification, + }, + { + skipReason: SkipReason.InvalidValue, + }, + { + skipReason: SkipReason.InvalidValue, + }, + ], + }); + }); + + it('extracts newName', () => { + expect(extractPackageFile(newName)).toMatchSnapshot({ + deps: [ + { + depName: 'awesome/postgres', + currentDigest: postgresDigest, + currentValue: '11', + replaceString: `awesome/postgres:11@${postgresDigest}`, + }, + { + depName: 'awesome/postgres', + currentDigest: undefined, + currentValue: '11', + replaceString: 'awesome/postgres:11', + }, + { + depName: 'awesome/postgres', + currentDigest: postgresDigest, currentValue: undefined, + replaceString: `awesome/postgres@${postgresDigest}`, }, ], }); diff --git a/lib/manager/kustomize/extract.ts b/lib/manager/kustomize/extract.ts index 8a06c3da71caf7..f4cd424f8bd663 100644 --- a/lib/manager/kustomize/extract.ts +++ b/lib/manager/kustomize/extract.ts @@ -5,7 +5,7 @@ import * as datasourceGitTags from '../../datasource/git-tags'; import * as datasourceGitHubTags from '../../datasource/github-tags'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; -import * as dockerVersioning from '../../versioning/docker'; +import { splitImageParts } from '../dockerfile/extract'; import type { PackageDependency, PackageFile } from '../types'; import type { Image, Kustomize } from './types'; @@ -21,7 +21,8 @@ export function extractBase(base: string): PackageDependency | null { return null; } - if (match?.groups.path.startsWith('github.com')) { + const { path } = match.groups; + if (path.startsWith('github.com:') || path.startsWith('github.com/')) { return { currentValue: match.groups.currentValue, datasource: datasourceGitHubTags.id, @@ -31,37 +32,72 @@ export function extractBase(base: string): PackageDependency | null { return { datasource: datasourceGitTags.id, - depName: match.groups.path.replace('.git', ''), + depName: path.replace('.git', ''), lookupName: match.groups.url, currentValue: match.groups.currentValue, }; } export function extractImage(image: Image): PackageDependency | null { - if (image?.name && image.newTag) { - const replaceString = image.newTag; - let currentValue: string | undefined; - let currentDigest: string | undefined; - if (!is.string(replaceString)) { + if (!image.name) { + return null; + } + const nameDep = splitImageParts(image.newName ?? image.name); + const { depName } = nameDep; + const { digest, newTag } = image; + if (digest && newTag) { + logger.warn( + { newTag, digest }, + 'Kustomize ignores newTag when digest is provided. Pick one, or use `newTag: tag@digest`' + ); + return { + depName, + currentValue: newTag, + currentDigest: digest, + skipReason: SkipReason.InvalidDependencySpecification, + }; + } + + if (digest) { + if (!is.string(digest) || !digest.startsWith('sha256:')) { return { - depName: image.newName ?? image.name, - currentValue: replaceString, + depName, + currentValue: digest, skipReason: SkipReason.InvalidValue, }; } - if (replaceString.startsWith('sha256:')) { - currentDigest = replaceString; - currentValue = undefined; - } else { - currentValue = replaceString; + + return { + datasource: datasourceDocker.id, + depName, + currentValue: nameDep.currentValue, + currentDigest: digest, + replaceString: digest, + }; + } + + if (newTag) { + if (!is.string(newTag) || newTag.startsWith('sha256:')) { + return { + depName, + currentValue: newTag, + skipReason: SkipReason.InvalidValue, + }; } + + const dep = splitImageParts(`${depName}:${newTag}`); + return { + ...dep, + datasource: datasourceDocker.id, + replaceString: newTag, + }; + } + + if (image.newName) { return { + ...nameDep, datasource: datasourceDocker.id, - versioning: dockerVersioning.id, - depName: image.newName ?? image.name, - currentValue, - currentDigest, - replaceString, + replaceString: image.newName, }; } diff --git a/lib/manager/kustomize/readme.md b/lib/manager/kustomize/readme.md index 1a63c7718cebf2..3753545eb43b90 100644 --- a/lib/manager/kustomize/readme.md +++ b/lib/manager/kustomize/readme.md @@ -15,16 +15,51 @@ This package will manage two parts of the `kustomization.yaml` file: - Needs to have `kind: Kustomization` defined - Currently this hasn't been tested using HTTPS to fetch the repos -- The image tags are limited to the following formats: +- The keys for the image tags can be in any order -``` +```yaml - name: image/name newTag: v0.0.1 +# or +- newTag: v0.0.1 + name: image/name ``` -or +- Digests can be pinned in `newTag` or `digest`: +```yaml +- name: image/name + newTag: v0.0.1@sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +# without a version, digests are tracked as :latest +- name: image/name + digest: sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f ``` -- newTag: v0.0.1 - name: image/name + +- The image's repository can be changed with `newName`: + +```yaml +- name: image/name + newName: custom-image/name:v0.0.1 +- name: image/name + newName: custom-image/name:v0.0.1@sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +- name: image/name + newName: custom-image/name@sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +- name: image/name + newName: custom-image/name + newTag: v0.0.1@sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +- name: image/name + newName: custom-image/name + digest: sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +``` + +- Images with values ignored by Kustomize will be skipped to avoid ambiguity: + +```yaml +# bad: skipped because newTag: is ignored when digest: is set +- name: image/name + newTag: v0.0.1 + digest: sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f +# good: +- name: image/name + newTag: v0.0.1@sha256:3eeba3e2caa30d2aba0fd78a34c1bbeebaa1b96c7aa3c95ec9bac44163c5ca4f ``` diff --git a/lib/manager/kustomize/types.ts b/lib/manager/kustomize/types.ts index b791575c30c056..8693cd568ebcb2 100644 --- a/lib/manager/kustomize/types.ts +++ b/lib/manager/kustomize/types.ts @@ -2,6 +2,7 @@ export interface Image { name: string; newTag: string; newName?: string; + digest?: string; } export interface Kustomize { kind: string;