From 06c3af21bd0b0f8a0a05196a10b0aedca9f30c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Thu, 18 May 2023 01:33:16 +0300 Subject: [PATCH 01/10] feat(flux): Support OCI Helm repositories --- lib/modules/manager/flux/extract.ts | 18 +++++++++++++++++- lib/modules/manager/flux/types.ts | 1 + 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index a15f5aa486fba1..d5237cbb062c70 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -3,6 +3,7 @@ import { logger } from '../../../logger'; import { readLocalFile } from '../../../util/fs'; import { regEx } from '../../../util/regex'; import { BitbucketTagsDatasource } from '../../datasource/bitbucket-tags'; +import { DockerDatasource } from '../../datasource/docker'; import { GitRefsDatasource } from '../../datasource/git-refs'; import { GitTagsDatasource } from '../../datasource/git-tags'; import { GithubReleasesDatasource } from '../../datasource/github-releases'; @@ -181,7 +182,22 @@ function resolveResourceManifest( resource.metadata?.namespace) ); if (matchingRepositories.length) { - dep.registryUrls = matchingRepositories.map((repo) => repo.spec.url); + dep.registryUrls = matchingRepositories.map((repo) => { + if (repo.spec.type === 'oci') { + // Change datasource to Docker + dep.datasource = DockerDatasource.id; + // Ensure the URL is a valid OCI path + dep.depName = `${repo.spec.url.replace( + 'oci://', + '' + )}/${dep.depName!}`; + return dep.depName; + } else if (repo.spec.url.startsWith('oci://')) { + return repo.spec.url.replace('oci://', ''); + } else { + return repo.spec.url; + } + }); } else { dep.skipReason = 'unknown-registry'; } diff --git a/lib/modules/manager/flux/types.ts b/lib/modules/manager/flux/types.ts index 7b18af8fdcddb0..c7848f84847d85 100644 --- a/lib/modules/manager/flux/types.ts +++ b/lib/modules/manager/flux/types.ts @@ -34,6 +34,7 @@ export interface HelmRepository extends KubernetesResource { kind: 'HelmRepository'; spec: { url: string; + type: string; }; } From 1844c5c7cf65b89668a49e1992222ad05d9e4848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Thu, 18 May 2023 01:50:31 +0300 Subject: [PATCH 02/10] feat(flux): OCI Helm repository test --- .../flux/__fixtures__/helmOCIRelease.yaml | 35 +++++++++++++++++++ .../flux/__fixtures__/helmOCISource.yaml | 10 ++++++ lib/modules/manager/flux/extract.spec.ts | 24 +++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml create mode 100644 lib/modules/manager/flux/__fixtures__/helmOCISource.yaml diff --git a/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml b/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml new file mode 100644 index 00000000000000..ce6267b9c2b8cc --- /dev/null +++ b/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml @@ -0,0 +1,35 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: arc-assets + namespace: dev +spec: + interval: 30m + chart: + spec: + chart: actions-runner-controller-charts/gha-runner-scale-set + version: 0.4.0 + sourceRef: + kind: HelmRepository + name: actions-runner-controller + namespace: flux-system + interval: 30m + + values: + runnerScaleSetName: arc-runner-set-assets + + githubConfigUrl: https://github.com/bjw-s/assets + + minRunners: 0 + maxRunners: 2 + + containerMode: + type: "dind" + + template: + spec: + containers: + - name: runner + image: ghcr.io/bjw-s/github-actions-runner:2.304.0@sha256:85690088155072a95c429a1f150f80a7d7ebf35107221cb25c01c56b62866c1b + command: + - /home/runner/run.sh diff --git a/lib/modules/manager/flux/__fixtures__/helmOCISource.yaml b/lib/modules/manager/flux/__fixtures__/helmOCISource.yaml new file mode 100644 index 00000000000000..a96a5a406f8692 --- /dev/null +++ b/lib/modules/manager/flux/__fixtures__/helmOCISource.yaml @@ -0,0 +1,10 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: actions-runner-controller + namespace: flux-system +spec: + type: oci + interval: 30m + url: oci://ghcr.io/actions + timeout: 3m diff --git a/lib/modules/manager/flux/extract.spec.ts b/lib/modules/manager/flux/extract.spec.ts index f14307c3cc62c9..6fe82463e8b366 100644 --- a/lib/modules/manager/flux/extract.spec.ts +++ b/lib/modules/manager/flux/extract.spec.ts @@ -728,6 +728,30 @@ describe('modules/manager/flux/extract', () => { ]); }); + it('extract oci helm', async () => { + const result = await extractAllPackageFiles(config, [ + 'lib/modules/manager/flux/__fixtures__/helmOCISource.yaml', + 'lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml', + ]); + expect(result).toEqual([ + { + deps: [ + { + currentValue: '0.4.0', + datasource: DockerDatasource.id, + depName: + 'ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set', + registryUrls: [ + 'ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set', + ], + }, + ], + packageFile: + 'lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml', + }, + ]); + }); + it('ignores files that do not exist', async () => { const result = await extractAllPackageFiles(config, [ 'lib/modules/manager/flux/__fixtures__/bogus.yaml', From a90588101c99f0092c6d49ec1825f60a8f3d9eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Thu, 18 May 2023 02:15:31 +0300 Subject: [PATCH 03/10] Flux: README update regarding OCI HelmRepository support. --- lib/modules/manager/flux/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/flux/readme.md b/lib/modules/manager/flux/readme.md index f8d9b60bd96bf2..55af102a1d4998 100644 --- a/lib/modules/manager/flux/readme.md +++ b/lib/modules/manager/flux/readme.md @@ -9,8 +9,8 @@ This manager parses [Flux](https://fluxcd.io/) YAML manifests and supports: Extracts `helm` dependencies from `HelmRelease` resources. -The `flux` manager only extracts `helm` dependencies for `HelmRelease` resources linked to `HelmRepository` or `GitRepository` sources. -Renovate does not support OCI `HelmRepository` sources, those with `type: oci`. +The `flux` manager extracts `helm` dependencies for `HelmRelease` resources linked to `HelmRepository` or `GitRepository` sources. +Renovate does support OCI `HelmRepository` sources, those with `type: oci`. In addition, for the `flux` manager to properly link `HelmRelease` and `HelmRepository` resources, _both_ of the following conditions must be met: From 6e177a1cca8fcb7c13e022bd2aeeef27d7fc70eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Fri, 19 May 2023 17:03:33 +0300 Subject: [PATCH 04/10] Flux: Address PR comments --- .../flux/__fixtures__/helmOCIRelease.yaml | 19 ---------- lib/modules/manager/flux/extract.spec.ts | 7 ++-- lib/modules/manager/flux/extract.ts | 35 ++++++++++--------- 3 files changed, 22 insertions(+), 39 deletions(-) diff --git a/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml b/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml index ce6267b9c2b8cc..99eb6c33723aae 100644 --- a/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml +++ b/lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml @@ -14,22 +14,3 @@ spec: name: actions-runner-controller namespace: flux-system interval: 30m - - values: - runnerScaleSetName: arc-runner-set-assets - - githubConfigUrl: https://github.com/bjw-s/assets - - minRunners: 0 - maxRunners: 2 - - containerMode: - type: "dind" - - template: - spec: - containers: - - name: runner - image: ghcr.io/bjw-s/github-actions-runner:2.304.0@sha256:85690088155072a95c429a1f150f80a7d7ebf35107221cb25c01c56b62866c1b - command: - - /home/runner/run.sh diff --git a/lib/modules/manager/flux/extract.spec.ts b/lib/modules/manager/flux/extract.spec.ts index 6fe82463e8b366..2816de6bd6e681 100644 --- a/lib/modules/manager/flux/extract.spec.ts +++ b/lib/modules/manager/flux/extract.spec.ts @@ -739,11 +739,10 @@ describe('modules/manager/flux/extract', () => { { currentValue: '0.4.0', datasource: DockerDatasource.id, - depName: + depName: 'actions-runner-controller-charts/gha-runner-scale-set', + packageName: 'ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set', - registryUrls: [ - 'ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set', - ], + registryUrls: [], }, ], packageFile: diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index d5237cbb062c70..0457daaab9b94b 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -1,3 +1,4 @@ +import is from '@sindresorhus/is'; import { loadAll } from 'js-yaml'; import { logger } from '../../../logger'; import { readLocalFile } from '../../../util/fs'; @@ -182,22 +183,24 @@ function resolveResourceManifest( resource.metadata?.namespace) ); if (matchingRepositories.length) { - dep.registryUrls = matchingRepositories.map((repo) => { - if (repo.spec.type === 'oci') { - // Change datasource to Docker - dep.datasource = DockerDatasource.id; - // Ensure the URL is a valid OCI path - dep.depName = `${repo.spec.url.replace( - 'oci://', - '' - )}/${dep.depName!}`; - return dep.depName; - } else if (repo.spec.url.startsWith('oci://')) { - return repo.spec.url.replace('oci://', ''); - } else { - return repo.spec.url; - } - }); + dep.registryUrls = matchingRepositories + .map((repo) => { + if (repo.spec.type === 'oci') { + // Change datasource to Docker + dep.datasource = DockerDatasource.id; + // Ensure the URL is a valid OCI path + dep.packageName = `${repo.spec.url.replace( + 'oci://', + '' + )}/${dep.depName!}`; + return null; + } else if (repo.spec.url.startsWith('oci://')) { + return repo.spec.url.replace('oci://', ''); + } else { + return repo.spec.url; + } + }) + .filter(is.string); } else { dep.skipReason = 'unknown-registry'; } From fba139f55a64f43186efc4797f2d488eeca8026d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Fri, 19 May 2023 20:29:39 +0300 Subject: [PATCH 05/10] Flux: Handle the case where OCI url is set but type is not. --- .../flux/__fixtures__/helmOCIRelease2.yaml | 19 +++++++++++++++ .../flux/__fixtures__/helmOCISource2.yaml | 8 +++++++ lib/modules/manager/flux/extract.spec.ts | 24 ++++++++++++++++++- lib/modules/manager/flux/extract.ts | 7 +++--- 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml create mode 100644 lib/modules/manager/flux/__fixtures__/helmOCISource2.yaml diff --git a/lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml b/lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml new file mode 100644 index 00000000000000..411497075bd0d3 --- /dev/null +++ b/lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml @@ -0,0 +1,19 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: kyverno + namespace: flux-system +spec: + interval: 6h + releaseName: kyverno + targetNamespace: kyverno + install: + createNamespace: true + chart: + spec: + chart: kyverno + version: 2.6.0 + interval: 6h + sourceRef: + kind: HelmRepository + name: kyverno diff --git a/lib/modules/manager/flux/__fixtures__/helmOCISource2.yaml b/lib/modules/manager/flux/__fixtures__/helmOCISource2.yaml new file mode 100644 index 00000000000000..7588d8f6810f18 --- /dev/null +++ b/lib/modules/manager/flux/__fixtures__/helmOCISource2.yaml @@ -0,0 +1,8 @@ +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: kyverno + namespace: flux-system +spec: + interval: 6h + url: oci://ghcr.io/kyverno/charts diff --git a/lib/modules/manager/flux/extract.spec.ts b/lib/modules/manager/flux/extract.spec.ts index 2816de6bd6e681..c674d9d8f39526 100644 --- a/lib/modules/manager/flux/extract.spec.ts +++ b/lib/modules/manager/flux/extract.spec.ts @@ -728,7 +728,7 @@ describe('modules/manager/flux/extract', () => { ]); }); - it('extract oci helm', async () => { + it('should handle HelmRepository with type OCI', async () => { const result = await extractAllPackageFiles(config, [ 'lib/modules/manager/flux/__fixtures__/helmOCISource.yaml', 'lib/modules/manager/flux/__fixtures__/helmOCIRelease.yaml', @@ -751,6 +751,28 @@ describe('modules/manager/flux/extract', () => { ]); }); + it('should handle HelmRepository w/o type oci and url starts with oci', async () => { + const result = await extractAllPackageFiles(config, [ + 'lib/modules/manager/flux/__fixtures__/helmOCISource2.yaml', + 'lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml', + ]); + expect(result).toEqual([ + { + deps: [ + { + currentValue: '2.6.0', + datasource: DockerDatasource.id, + depName: 'kyverno', + packageName: 'ghcr.io/kyverno/charts/kyverno', + registryUrls: [], + }, + ], + packageFile: + 'lib/modules/manager/flux/__fixtures__/helmOCIRelease2.yaml', + }, + ]); + }); + it('ignores files that do not exist', async () => { const result = await extractAllPackageFiles(config, [ 'lib/modules/manager/flux/__fixtures__/bogus.yaml', diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index 0457daaab9b94b..e26f2e49878663 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -185,7 +185,10 @@ function resolveResourceManifest( if (matchingRepositories.length) { dep.registryUrls = matchingRepositories .map((repo) => { - if (repo.spec.type === 'oci') { + if ( + repo.spec.type === 'oci' || + repo.spec.url.startsWith('oci://') + ) { // Change datasource to Docker dep.datasource = DockerDatasource.id; // Ensure the URL is a valid OCI path @@ -194,8 +197,6 @@ function resolveResourceManifest( '' )}/${dep.depName!}`; return null; - } else if (repo.spec.url.startsWith('oci://')) { - return repo.spec.url.replace('oci://', ''); } else { return repo.spec.url; } From 1ad8640b3ac016df4ff58497e740abead5c88974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Fri, 19 May 2023 21:15:40 +0300 Subject: [PATCH 06/10] Flux: declared allowed HelmRepository types. --- lib/modules/manager/flux/types.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/flux/types.ts b/lib/modules/manager/flux/types.ts index c7848f84847d85..9895abc28b29e6 100644 --- a/lib/modules/manager/flux/types.ts +++ b/lib/modules/manager/flux/types.ts @@ -30,11 +30,13 @@ export interface HelmRelease extends KubernetesResource { }; } +export type HelmRepositoryType = 'oci' | 'default'; + export interface HelmRepository extends KubernetesResource { kind: 'HelmRepository'; spec: { url: string; - type: string; + type: HelmRepositoryType; }; } From 4e0c80c0d3784c489cc6bd13e0c2c6a06ce21079 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Fri, 19 May 2023 21:20:25 +0300 Subject: [PATCH 07/10] Flux: delete registry urls when empty and fix tests after --- lib/modules/manager/flux/extract.spec.ts | 2 -- lib/modules/manager/flux/extract.ts | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/flux/extract.spec.ts b/lib/modules/manager/flux/extract.spec.ts index c674d9d8f39526..2775ec92127646 100644 --- a/lib/modules/manager/flux/extract.spec.ts +++ b/lib/modules/manager/flux/extract.spec.ts @@ -742,7 +742,6 @@ describe('modules/manager/flux/extract', () => { depName: 'actions-runner-controller-charts/gha-runner-scale-set', packageName: 'ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set', - registryUrls: [], }, ], packageFile: @@ -764,7 +763,6 @@ describe('modules/manager/flux/extract', () => { datasource: DockerDatasource.id, depName: 'kyverno', packageName: 'ghcr.io/kyverno/charts/kyverno', - registryUrls: [], }, ], packageFile: diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index e26f2e49878663..65994e015f31c3 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -202,6 +202,11 @@ function resolveResourceManifest( } }) .filter(is.string); + + // if registryUrls is empty, delete it from dep + if (dep.registryUrls && dep.registryUrls.length === 0) { + delete dep.registryUrls; + } } else { dep.skipReason = 'unknown-registry'; } From 94bebcb02fcbeedcf0e22a75a3f6bc6b563508c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Sun, 21 May 2023 21:49:11 +0300 Subject: [PATCH 08/10] Flux: Simplify as requested Co-authored-by: Michael Kriese --- lib/modules/manager/flux/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index 65994e015f31c3..a5f041efdea2cd 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -204,7 +204,7 @@ function resolveResourceManifest( .filter(is.string); // if registryUrls is empty, delete it from dep - if (dep.registryUrls && dep.registryUrls.length === 0) { + if (!dep.registryUrls?.length) { delete dep.registryUrls; } } else { From 2ae8a42a06c06b1a70bcd52d9f3b8dfa5605af99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Thu, 25 May 2023 06:16:18 +0300 Subject: [PATCH 09/10] Flux: Use resrouce chart value instead of dep.depName. --- lib/modules/manager/flux/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index a5f041efdea2cd..f69f9b05082070 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -195,7 +195,7 @@ function resolveResourceManifest( dep.packageName = `${repo.spec.url.replace( 'oci://', '' - )}/${dep.depName!}`; + )}/${resource.spec.chart.spec.chart}`; return null; } else { return repo.spec.url; From 80840d6c5ec0b7b865320a8387386adef6cfdef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Skyler=20M=C3=A4ntysaari?= Date: Thu, 25 May 2023 06:51:44 +0300 Subject: [PATCH 10/10] Flux: Prettier --- lib/modules/manager/flux/extract.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/modules/manager/flux/extract.ts b/lib/modules/manager/flux/extract.ts index f69f9b05082070..5713498c6999f1 100644 --- a/lib/modules/manager/flux/extract.ts +++ b/lib/modules/manager/flux/extract.ts @@ -192,10 +192,9 @@ function resolveResourceManifest( // Change datasource to Docker dep.datasource = DockerDatasource.id; // Ensure the URL is a valid OCI path - dep.packageName = `${repo.spec.url.replace( - 'oci://', - '' - )}/${resource.spec.chart.spec.chart}`; + dep.packageName = `${repo.spec.url.replace('oci://', '')}/${ + resource.spec.chart.spec.chart + }`; return null; } else { return repo.spec.url;