From 84e31d4f0015d8b892b5738d9c74c7a8ded6bc9e Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Mon, 9 Oct 2023 19:54:09 -0400 Subject: [PATCH 01/52] feat(vendir): provides support for vendir manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently only supports the helmChart dependency, but provides a framework for other types of dependencies in vendir Co-authored-by: Zoltán Reegn --- lib/modules/manager/api.ts | 2 + lib/modules/manager/vendir/README.md | 27 ++++++ .../vendir/__fixtures__/empty-directory.yaml | 3 + .../__fixtures__/multiple-helm-chart.yaml | 17 ++++ .../vendir/__fixtures__/one-helm-chart.yaml | 11 +++ lib/modules/manager/vendir/extract.spec.ts | 63 ++++++++++++++ lib/modules/manager/vendir/extract.ts | 86 +++++++++++++++++++ lib/modules/manager/vendir/index.ts | 10 +++ lib/modules/manager/vendir/types.ts | 24 ++++++ package.json | 2 +- 10 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 lib/modules/manager/vendir/README.md create mode 100644 lib/modules/manager/vendir/__fixtures__/empty-directory.yaml create mode 100644 lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml create mode 100644 lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml create mode 100644 lib/modules/manager/vendir/extract.spec.ts create mode 100644 lib/modules/manager/vendir/extract.ts create mode 100644 lib/modules/manager/vendir/index.ts create mode 100644 lib/modules/manager/vendir/types.ts diff --git a/lib/modules/manager/api.ts b/lib/modules/manager/api.ts index c5cbbea651b770..0f38e101092445 100644 --- a/lib/modules/manager/api.ts +++ b/lib/modules/manager/api.ts @@ -84,6 +84,7 @@ import * as tflintPlugin from './tflint-plugin'; import * as travis from './travis'; import type { ManagerApi } from './types'; import * as velaci from './velaci'; +import * as vendir from './vendir'; import * as woodpecker from './woodpecker'; const api = new Map(); @@ -174,4 +175,5 @@ api.set('terragrunt-version', terragruntVersion); api.set('tflint-plugin', tflintPlugin); api.set('travis', travis); api.set('velaci', velaci); +api.set('vendir', vendir); api.set('woodpecker', woodpecker); diff --git a/lib/modules/manager/vendir/README.md b/lib/modules/manager/vendir/README.md new file mode 100644 index 00000000000000..87ce863961ec58 --- /dev/null +++ b/lib/modules/manager/vendir/README.md @@ -0,0 +1,27 @@ +The [vendir](https://carvel.dev/vendir/) manager only updates dependencies of the `helmChart` type. + +```yaml +# fetch Helm chart contents (optional; v0.11.0+) +helmChart: + # chart name (required) + name: stable/redis + # use specific chart version (string; optional) + version: "1.2.1" + # specifies Helm repository to fetch from (optional) + repository: + # repository url; supports exprimental oci helm fetch via + # oci:// scheme (required) + url: https://... + # specifies name of a secret with helm repo auth details; + # secret may include 'username', 'password'; + # as of v0.19.0+, dockerconfigjson secrets are also supported (optional) + # as of v0.22.0+, 0 or 1 auth credential is expected within dockerconfigjson secret + # if >1 auth creds found, error will be returned. (currently registry hostname + # is not used when found in provide auth credential.) + secretRef: + # (required) + name: my-helm-auth + # specify helm binary version to use; + # '3' means binary 'helm3' needs to be on the path (optional) + helmVersion: "3" +``` diff --git a/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml b/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml new file mode 100644 index 00000000000000..b9e824668b4657 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml @@ -0,0 +1,3 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: [] diff --git a/lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml b/lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml new file mode 100644 index 00000000000000..05ecd9fb0e2963 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml @@ -0,0 +1,17 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: custom-repo-custom-version + helmChart: + name: contour + version: "7.10.1" + repository: + url: https://charts.bitnami.com/bitnami + - path: thing + helmChart: + name: contour + version: "7.10.1" + repository: + url: https://charts.bitnami.com/bitnami diff --git a/lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml b/lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml new file mode 100644 index 00000000000000..db27125f6aae48 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml @@ -0,0 +1,11 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: custom-repo-custom-version + helmChart: + name: contour + version: "7.10.1" + repository: + url: https://charts.bitnami.com/bitnami diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts new file mode 100644 index 00000000000000..31b02d4bf14110 --- /dev/null +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -0,0 +1,63 @@ +import { Fixtures } from '../../../../test/fixtures'; +import { extractPackageFile } from '.'; + +const oneHelmChart = Fixtures.get('one-helm-chart.yaml'); +const multipleHelmChart = Fixtures.get('multiple-helm-chart.yaml'); +const emptyDirectories = Fixtures.get('empty-directory.yaml'); + +describe('modules/manager/vendir/extract', () => { + describe('extractPackageFile()', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('returns null for invalid yaml file content', () => { + const result = extractPackageFile('nothing here: ['); + expect(result).toBeNull(); + }); + + it('returns null for empty yaml file content', () => { + const result = extractPackageFile(''); + expect(result).toBeNull(); + }); + + it('returns null for empty directories key', () => { + const result = extractPackageFile(emptyDirectories); + expect(result).toBeNull(); + }); + + it('single chart - extracts helm-chart from vendir.yml correctly', () => { + const result = extractPackageFile(oneHelmChart); + expect(result).toMatchObject({ + deps: [ + { + currentValue: '7.10.1', + depName: 'contour', + datasource: 'helm', + registryUrls: ['https://charts.bitnami.com/bitnami'], + }, + ], + }); + }); + + it('multiple charts - extracts helm-chart from vendir.yml correctly', () => { + const result = extractPackageFile(multipleHelmChart); + expect(result).toMatchObject({ + deps: [ + { + currentValue: '7.10.1', + depName: 'contour', + datasource: 'helm', + registryUrls: ['https://charts.bitnami.com/bitnami'], + }, + { + currentValue: '7.10.1', + depName: 'contour', + datasource: 'helm', + registryUrls: ['https://charts.bitnami.com/bitnami'], + }, + ], + }); + }); + }); +}); diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts new file mode 100644 index 00000000000000..f61e25dd701873 --- /dev/null +++ b/lib/modules/manager/vendir/extract.ts @@ -0,0 +1,86 @@ +import is from '@sindresorhus/is'; +import { load } from 'js-yaml'; +import { logger } from '../../../logger'; +import { HelmDatasource } from '../../datasource/helm'; +import type { PackageDependency, PackageFileContent } from '../types'; +import type { HelmChart, Vendir } from './types'; + +// TODO: Add support for other vendir types (like git tags, github releases, etc.) +// Recommend looking at the kustomize manager for more information on support. + +export function extractHelmChart( + helmChart: HelmChart +): PackageDependency | null { + if (!helmChart.name) { + return null; + } + + return { + depName: helmChart.name, + currentValue: helmChart.version, + registryUrls: [helmChart.repository.url], + datasource: HelmDatasource.id, + }; +} + +export function parseVendir( + content: string, + packageFile?: string +): Vendir | null { + let pkg: Vendir | null = null; + try { + pkg = load(content, { json: true }) as Vendir; + } catch (e) /* istanbul ignore next */ { + logger.debug({ packageFile }, 'Error parsing vendir.yml file'); + return null; + } + + if (!pkg || is.string(pkg)) { + return null; + } + + pkg.kind ??= 'Config'; + + if (!['Config'].includes(pkg.kind)) { + return null; + } + + return pkg; +} + +export function extractPackageFile( + content: string, + packageFile?: string // TODO: fix tests +): PackageFileContent | null { + logger.trace(`vendir.extractPackageFile(${packageFile!})`); + const deps: PackageDependency[] = []; + + const pkg = parseVendir(content, packageFile); + if (!pkg) { + return null; + } + + // grab the helm charts + // TODO: Add support for OCI Repos by translating Registry URLs and using + // Docker datasource. (See Helmv3 for example implementation) + const contents = pkg.directories.flatMap((directory) => directory.contents); + const charts = contents.filter( + (chart) => typeof chart.helmChart === 'object' + ); + charts.forEach((chart) => { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const dep = extractHelmChart(chart.helmChart); + if (dep) { + deps.push({ + ...dep, + depType: 'HelmChart', + }); + } + }); + + if (!deps.length) { + return null; + } + return { deps }; +} diff --git a/lib/modules/manager/vendir/index.ts b/lib/modules/manager/vendir/index.ts new file mode 100644 index 00000000000000..24ec53eed196b9 --- /dev/null +++ b/lib/modules/manager/vendir/index.ts @@ -0,0 +1,10 @@ +import { HelmDatasource } from '../../datasource/helm'; +export { extractPackageFile } from './extract'; + +export const defaultConfig = { + commitMessageTopic: 'vendir {{depName}}', + fileMatch: ['(^|/)vendir\\.yml$'], +}; + +export const supportedDatasources = [HelmDatasource.id]; +export const supportsLockFileMaintenance = true; diff --git a/lib/modules/manager/vendir/types.ts b/lib/modules/manager/vendir/types.ts new file mode 100644 index 00000000000000..957ff6e9acc2f8 --- /dev/null +++ b/lib/modules/manager/vendir/types.ts @@ -0,0 +1,24 @@ +export interface Vendir { + kind?: string; + directories: Directories[]; +} + +export interface Directories { + path: string; + contents: Contents[]; +} + +export interface Contents { + path: string; + helmChart?: HelmChart; +} + +export interface HelmChart { + name: string; + version: string; + repository: Repository; +} + +export interface Repository { + url: string; +} diff --git a/package.json b/package.json index 65cbbec7e09ae1..bee23cc7fc5788 100644 --- a/package.json +++ b/package.json @@ -143,12 +143,12 @@ "node": "^18.12.0 || >=20.0.0" }, "dependencies": { - "@aws-sdk/credential-providers": "3.363.0", "@aws-sdk/client-codecommit": "3.363.0", "@aws-sdk/client-ec2": "3.363.0", "@aws-sdk/client-ecr": "3.363.0", "@aws-sdk/client-rds": "3.363.0", "@aws-sdk/client-s3": "3.363.0", + "@aws-sdk/credential-providers": "3.363.0", "@breejs/later": "4.1.0", "@cdktf/hcl2json": "0.18.0", "@iarna/toml": "3.0.0", From 85585b470e5dba40d61d497f805bf335bdcf9cd3 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Mon, 9 Oct 2023 20:39:04 -0400 Subject: [PATCH 02/52] feat(vendir): add support for running vendir sync --- .../manager/vendir/__fixtures__/vendir.yml | 15 + .../manager/vendir/__fixtures__/vendir_1.lock | 9 + .../manager/vendir/__fixtures__/vendir_2.lock | 9 + .../__snapshots__/artifacts.spec.ts.snap | 24 + lib/modules/manager/vendir/artifacts.spec.ts | 506 ++++++++++++++++++ lib/modules/manager/vendir/artifacts.ts | 139 +++++ lib/modules/manager/vendir/index.ts | 1 + lib/modules/manager/vendir/types.ts | 6 + lib/modules/manager/vendir/utils.ts | 7 + 9 files changed, 716 insertions(+) create mode 100644 lib/modules/manager/vendir/__fixtures__/vendir.yml create mode 100644 lib/modules/manager/vendir/__fixtures__/vendir_1.lock create mode 100644 lib/modules/manager/vendir/__fixtures__/vendir_2.lock create mode 100644 lib/modules/manager/vendir/__snapshots__/artifacts.spec.ts.snap create mode 100644 lib/modules/manager/vendir/artifacts.spec.ts create mode 100644 lib/modules/manager/vendir/artifacts.ts create mode 100644 lib/modules/manager/vendir/utils.ts diff --git a/lib/modules/manager/vendir/__fixtures__/vendir.yml b/lib/modules/manager/vendir/__fixtures__/vendir.yml new file mode 100644 index 00000000000000..ca496ff089bf08 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/vendir.yml @@ -0,0 +1,15 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config + +minimumRequiredVersion: 0.32.0 + +# one or more directories to manage with vendir +directories: + - path: vendor + contents: + - path: renovate + helmChart: + name: renovate + version: 36.109.4 + repository: + url: https://docs.renovatebot.com/helm-charts diff --git a/lib/modules/manager/vendir/__fixtures__/vendir_1.lock b/lib/modules/manager/vendir/__fixtures__/vendir_1.lock new file mode 100644 index 00000000000000..49242c2e9efae3 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/vendir_1.lock @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - helmChart: + appVersion: 36.109.4 + version: 36.109.4 + path: renovate + path: vendor +kind: LockConfig diff --git a/lib/modules/manager/vendir/__fixtures__/vendir_2.lock b/lib/modules/manager/vendir/__fixtures__/vendir_2.lock new file mode 100644 index 00000000000000..76db684b7cd8bf --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/vendir_2.lock @@ -0,0 +1,9 @@ +apiVersion: vendir.k14s.io/v1alpha1 +directories: +- contents: + - helmChart: + appVersion: 36.109.4 + version: 37.109.4 + path: renovate + path: vendor +kind: LockConfig diff --git a/lib/modules/manager/vendir/__snapshots__/artifacts.spec.ts.snap b/lib/modules/manager/vendir/__snapshots__/artifacts.spec.ts.snap new file mode 100644 index 00000000000000..059191726c0413 --- /dev/null +++ b/lib/modules/manager/vendir/__snapshots__/artifacts.spec.ts.snap @@ -0,0 +1,24 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`modules/manager/vendir/artifacts returns null if unchanged 1`] = ` +[ + { + "cmd": "vendir sync", + "options": { + "cwd": "/tmp/github/some/repo", + "encoding": "utf-8", + "env": { + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + }, + "maxBuffer": 10485760, + "timeout": 900000, + }, + }, +] +`; diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts new file mode 100644 index 00000000000000..6170e320556ba4 --- /dev/null +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -0,0 +1,506 @@ +import { mockDeep } from 'jest-mock-extended'; +import { join } from 'upath'; +import { envMock, mockExecAll } from '../../../../test/exec-util'; +import { Fixtures } from '../../../../test/fixtures'; +import { env, fs, git, partial } from '../../../../test/util'; +import { GlobalConfig } from '../../../config/global'; +import type { RepoGlobalConfig } from '../../../config/types'; +import type { StatusResult } from '../../../util/git/types'; +import type { UpdateArtifactsConfig } from '../types'; +import * as vendir from '.'; + +jest.mock('../../datasource', () => mockDeep()); +jest.mock('../../../util/exec/env'); +jest.mock('../../../util/http'); +jest.mock('../../../util/fs'); +jest.mock('../../../util/git'); + +const adminConfig: RepoGlobalConfig = { + localDir: join('/tmp/github/some/repo'), // `join` fixes Windows CI + cacheDir: join('/tmp/renovate/cache'), + containerbaseDir: join('/tmp/renovate/cache/containerbase'), +}; + +const config: UpdateArtifactsConfig = {}; +const vendirLockFile1 = Fixtures.get('vendir_1.lock'); +const vendirLockFile2 = Fixtures.get('vendir_2.lock'); +const vendirFile = Fixtures.get('vendir.yml'); + +describe('modules/manager/vendir/artifacts', () => { + beforeEach(() => { + env.getChildProcessEnv.mockReturnValue(envMock.basic); + GlobalConfig.set(adminConfig); + }); + + afterEach(() => { + GlobalConfig.reset(); + }); + + it('returns null if no vendir.lock.yml found', async () => { + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: '', + config, + }) + ).toBeNull(); + }); + + it('returns null if updatedDeps is empty', async () => { + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.lock.yml', + updatedDeps: [], + newPackageFileContent: '', + config, + }) + ).toBeNull(); + }); + + it('returns null if unchanged', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + const execSnapshots = mockExecAll(); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }) + ).toBeNull(); + expect(execSnapshots).toMatchSnapshot(); + }); + + it('returns updated vendir.lock', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + const execSnapshots = mockExecAll(); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }) + ).toMatchObject([ + { + file: { + type: 'addition', + path: 'vendir.lock.yml', + contents: vendirLockFile2, + }, + }, + ]); + expect(execSnapshots).toBeArrayOfSize(2); + expect(execSnapshots).toMatchSnapshot(); + }); + + it('returns updated vendir.yml for lockfile maintenance', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps: [], + newPackageFileContent: vendirFile, + config: { ...config, updateType: 'lockFileMaintenance' }, + }) + ).toMatchObject([ + { + file: { + type: 'addition', + path: 'vendir.yml', + contents: vendirLockFile2, + }, + }, + ]); + expect(execSnapshots).toBeArrayOfSize(2); + expect(execSnapshots).toMatchSnapshot(); + }); + + it('catches errors', async () => { + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.writeLocalFile.mockImplementationOnce(() => { + throw new Error('not found'); + }); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }) + ).toMatchObject([ + { + artifactError: { + lockFile: 'vendir.yml', + stderr: 'not found', + }, + }, + ]); + }); + + it('add sub chart artifacts to file list if vendir.yml exists', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + + // sub chart artifacts + fs.getSiblingFileName.mockReturnValueOnce('charts'); + git.getRepoStatus.mockResolvedValueOnce( + partial({ + not_added: ['charts/example-1.9.2.tgz'], + deleted: ['charts/example-1.6.2.tgz'], + }) + ); + const updatedDeps = [{ depName: 'dep1' }]; + const test = await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config: { + postUpdateOptions: ['helmUpdateSubChartArchives'], + ...config, + }, + }); + expect(test).toEqual([ + { + file: { + type: 'addition', + path: 'vendir.yml', + contents: vendirLockFile2, + }, + }, + { + file: { + type: 'addition', + path: 'charts/example-1.9.2.tgz', + contents: undefined, + }, + }, + { + file: { + type: 'deletion', + path: 'charts/example-1.6.2.tgz', + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { + cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + }, + { + cmd: "helm dependency update ''", + }, + ]); + }); + + it('add sub chart artifacts to file list if vendir.yml is missing', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + + // sub chart artifacts + fs.getSiblingFileName.mockReturnValueOnce('charts'); + git.getRepoStatus.mockResolvedValueOnce( + partial({ + not_added: ['charts/example-1.9.2.tgz'], + deleted: ['charts/example-1.6.2.tgz'], + }) + ); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config: { + postUpdateOptions: ['helmUpdateSubChartArchives'], + ...config, + }, + }) + ).toEqual([ + { + file: { + type: 'addition', + path: 'charts/example-1.9.2.tgz', + contents: undefined, + }, + }, + { + file: { + type: 'deletion', + path: 'charts/example-1.6.2.tgz', + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { + cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + { + cmd: "helm dependency update ''", + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + ]); + }); + + it('add sub chart artifacts without old archives', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + + // sub chart artifacts + fs.getSiblingFileName.mockReturnValueOnce('charts'); + git.getRepoStatus.mockResolvedValueOnce( + partial({ + not_added: ['charts/example-1.9.2.tgz'], + }) + ); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config: { + postUpdateOptions: ['helmUpdateSubChartArchives'], + ...config, + }, + }) + ).toEqual([ + { + file: { + type: 'addition', + path: 'charts/example-1.9.2.tgz', + contents: undefined, + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { + cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + { + cmd: "helm dependency update ''", + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + ]); + }); + + it('add sub chart artifacts and ignore files outside of the chart folder', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + + // sub chart artifacts + fs.getSiblingFileName.mockReturnValueOnce('charts'); + git.getRepoStatus.mockResolvedValueOnce( + partial({ + not_added: ['charts/example-1.9.2.tgz', 'exampleFile'], + deleted: ['charts/example-1.6.2.tgz', 'aFolder/otherFile'], + }) + ); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config: { + postUpdateOptions: ['helmUpdateSubChartArchives'], + ...config, + }, + }) + ).toEqual([ + { + file: { + type: 'addition', + path: 'charts/example-1.9.2.tgz', + contents: undefined, + }, + }, + { + file: { + type: 'deletion', + path: 'charts/example-1.6.2.tgz', + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { + cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + { + cmd: "helm dependency update ''", + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + ]); + }); + + it('skip artifacts which are not lock files or in the chart folder', async () => { + fs.readLocalFile.mockResolvedValueOnce(null); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache' + ); + fs.getParentDir.mockReturnValue(''); + + // sub chart artifacts + fs.getSiblingFileName.mockReturnValueOnce('charts'); + git.getRepoStatus.mockResolvedValueOnce( + partial({ + modified: ['example/example.tgz'], + }) + ); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config: { + postUpdateOptions: ['helmUpdateSubChartArchives'], + ...config, + }, + }) + ).toBeNull(); + expect(execSnapshots).toMatchObject([ + { + cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + { + cmd: "helm dependency update ''", + options: { + env: { + HELM_EXPERIMENTAL_OCI: '1', + HELM_REGISTRY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/registry.json', + HELM_REPOSITORY_CONFIG: + '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', + HELM_REPOSITORY_CACHE: + '/tmp/renovate/cache/__renovate-private-cache/repositories', + }, + }, + }, + ]); + }); +}); diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts new file mode 100644 index 00000000000000..21a3990f560a17 --- /dev/null +++ b/lib/modules/manager/vendir/artifacts.ts @@ -0,0 +1,139 @@ +import is from '@sindresorhus/is'; +import { TEMPORARY_ERROR } from '../../../constants/error-messages'; +import { logger } from '../../../logger'; +import { exec } from '../../../util/exec'; +import type { ExecOptions, ToolConstraint } from '../../../util/exec/types'; +import { + getParentDir, + getSiblingFileName, + readLocalFile, + writeLocalFile, +} from '../../../util/fs'; +import { getRepoStatus } from '../../../util/git'; +import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; +import { isFileInDir } from './utils'; + +async function vendirCommands(execOptions: ExecOptions): Promise { + const cmd: string[] = []; + + cmd.push(`vendir sync`); + + await exec(cmd, execOptions); +} + +export async function updateArtifacts({ + packageFileName, + updatedDeps, + newPackageFileContent, + config, +}: UpdateArtifact): Promise { + logger.debug(`vendir.updateArtifacts(${packageFileName})`); + + const isLockFileMaintenance = config.updateType === 'lockFileMaintenance'; + const isUpdateOptionAddVendirArchives = config.postUpdateOptions?.includes( + 'vendirUpdateSubChartArchives' + ); + + if ( + !isLockFileMaintenance && + (updatedDeps === undefined || updatedDeps.length < 1) + ) { + logger.debug('No updated vendir deps - returning null'); + return null; + } + + const lockFileName = getSiblingFileName(packageFileName, 'vendir.lock.yml'); + const existingLockFileContent = await readLocalFile(lockFileName, 'utf8'); + if (!existingLockFileContent && !isUpdateOptionAddVendirArchives) { + logger.debug('No vendir.lock.yml found'); + return null; + } + try { + await writeLocalFile(packageFileName, newPackageFileContent); + logger.debug('Updating Vendir artifacts'); + const vendirToolConstraint: ToolConstraint = { + toolName: 'vendir', + constraint: config.constraints?.helm, + }; + + const execOptions: ExecOptions = { + cwdFile: packageFileName, + toolConstraints: [vendirToolConstraint], + }; + await vendirCommands(execOptions); + logger.debug('Returning updated Vendir artifacts'); + + const fileChanges: UpdateArtifactsResult[] = []; + + if (is.truthy(existingLockFileContent)) { + const newVendirLockContent = await readLocalFile(lockFileName, 'utf8'); + const isLockFileChanged = + existingLockFileContent !== newVendirLockContent; + if (isLockFileChanged) { + fileChanges.push({ + file: { + type: 'addition', + path: lockFileName, + contents: newVendirLockContent, + }, + }); + } else { + logger.debug('vendir.lock.yml is unchanged'); + } + } + + // add modified vendir archives to artifacts + if (is.truthy(isUpdateOptionAddVendirArchives)) { + const vendirPath = getSiblingFileName( + packageFileName, + getParentDir(packageFileName) + ); + const status = await getRepoStatus(); + const vendirAddition = status.not_added ?? []; + const vendirDeletion = status.deleted ?? []; + + for (const file of vendirAddition) { + // only add artifacts in the vendir sub path + if (!isFileInDir(vendirPath, file)) { + continue; + } + fileChanges.push({ + file: { + type: 'addition', + path: file, + contents: await readLocalFile(file), + }, + }); + } + + for (const file of vendirDeletion) { + // only add artifacts in the vendir sub path + if (!isFileInDir(vendirPath, file)) { + continue; + } + fileChanges.push({ + file: { + type: 'deletion', + path: file, + }, + }); + } + } + + return fileChanges.length > 0 ? fileChanges : null; + } catch (err) { + // istanbul ignore if + if (err.message === TEMPORARY_ERROR) { + throw err; + } + logger.debug({ err }, 'Failed to update Vendir lock file'); + return [ + { + artifactError: { + lockFile: lockFileName, + stderr: err.message, + }, + }, + ]; + } +} diff --git a/lib/modules/manager/vendir/index.ts b/lib/modules/manager/vendir/index.ts index 24ec53eed196b9..de9d67d3fc3400 100644 --- a/lib/modules/manager/vendir/index.ts +++ b/lib/modules/manager/vendir/index.ts @@ -1,5 +1,6 @@ import { HelmDatasource } from '../../datasource/helm'; export { extractPackageFile } from './extract'; +export { updateArtifacts } from './artifacts'; export const defaultConfig = { commitMessageTopic: 'vendir {{depName}}', diff --git a/lib/modules/manager/vendir/types.ts b/lib/modules/manager/vendir/types.ts index 957ff6e9acc2f8..fd91cb21ccc712 100644 --- a/lib/modules/manager/vendir/types.ts +++ b/lib/modules/manager/vendir/types.ts @@ -1,3 +1,5 @@ +import type { HostRule } from '../../../types'; + export interface Vendir { kind?: string; directories: Directories[]; @@ -22,3 +24,7 @@ export interface HelmChart { export interface Repository { url: string; } + +export interface RepositoryRule extends Repository { + hostRule: HostRule; +} diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts new file mode 100644 index 00000000000000..5930419fcf968b --- /dev/null +++ b/lib/modules/manager/vendir/utils.ts @@ -0,0 +1,7 @@ +import upath from 'upath'; + +export function isFileInDir(dir: string, file: string): boolean { + return upath.dirname(file) === dir; +} + +// TODO: Add Support for Registry Aliases (See Helmv3 for possible implementation) From 0f36351514a4c614027f1c1dab3f7f05c6e4baa2 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Tue, 10 Oct 2023 09:30:14 -0400 Subject: [PATCH 03/52] feat(vendir): properly run vendir sync feat(vendir): check types correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Zoltán Reegn --- ...helm-chart.yaml => multiple-contents.yaml} | 0 ...{one-helm-chart.yaml => one-contents.yaml} | 0 lib/modules/manager/vendir/artifacts.spec.ts | 262 ++++-------------- lib/modules/manager/vendir/artifacts.ts | 49 ++-- lib/modules/manager/vendir/extract.spec.ts | 8 +- lib/modules/manager/vendir/extract.ts | 24 +- lib/modules/manager/vendir/types.ts | 10 +- lib/modules/manager/vendir/utils.ts | 7 +- lib/util/exec/containerbase.ts | 5 + 9 files changed, 108 insertions(+), 257 deletions(-) rename lib/modules/manager/vendir/__fixtures__/{multiple-helm-chart.yaml => multiple-contents.yaml} (100%) rename lib/modules/manager/vendir/__fixtures__/{one-helm-chart.yaml => one-contents.yaml} (100%) diff --git a/lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml b/lib/modules/manager/vendir/__fixtures__/multiple-contents.yaml similarity index 100% rename from lib/modules/manager/vendir/__fixtures__/multiple-helm-chart.yaml rename to lib/modules/manager/vendir/__fixtures__/multiple-contents.yaml diff --git a/lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml b/lib/modules/manager/vendir/__fixtures__/one-contents.yaml similarity index 100% rename from lib/modules/manager/vendir/__fixtures__/one-helm-chart.yaml rename to lib/modules/manager/vendir/__fixtures__/one-contents.yaml diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 6170e320556ba4..765d095eca7fe5 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -97,7 +97,7 @@ describe('modules/manager/vendir/artifacts', () => { newPackageFileContent: vendirFile, config, }) - ).toMatchObject([ + ).toEqual([ { file: { type: 'addition', @@ -106,8 +106,7 @@ describe('modules/manager/vendir/artifacts', () => { }, }, ]); - expect(execSnapshots).toBeArrayOfSize(2); - expect(execSnapshots).toMatchSnapshot(); + expect(execSnapshots).toMatchObject([{ cmd: 'vendir sync' }]); }); it('returns updated vendir.yml for lockfile maintenance', async () => { @@ -135,8 +134,7 @@ describe('modules/manager/vendir/artifacts', () => { }, }, ]); - expect(execSnapshots).toBeArrayOfSize(2); - expect(execSnapshots).toMatchSnapshot(); + expect(execSnapshots).toMatchObject([{ cmd: 'vendir sync' }]); }); it('catches errors', async () => { @@ -166,9 +164,9 @@ describe('modules/manager/vendir/artifacts', () => { ]); }); - it('add sub chart artifacts to file list if vendir.yml exists', async () => { + it('add artifacts to file list if vendir.yml exists', async () => { fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); - fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); fs.privateCacheDir.mockReturnValue( @@ -176,12 +174,12 @@ describe('modules/manager/vendir/artifacts', () => { ); fs.getParentDir.mockReturnValue(''); - // sub chart artifacts - fs.getSiblingFileName.mockReturnValueOnce('charts'); + // artifacts + fs.getSiblingFileName.mockReturnValueOnce('vendor'); git.getRepoStatus.mockResolvedValueOnce( partial({ - not_added: ['charts/example-1.9.2.tgz'], - deleted: ['charts/example-1.6.2.tgz'], + not_added: ['vendor/Chart.yaml', 'vendor/my-chart/Chart.yaml'], + deleted: ['vendor/removed.yaml'], }) ); const updatedDeps = [{ depName: 'dep1' }]; @@ -190,7 +188,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['helmUpdateSubChartArchives'], + postUpdateOptions: ['vendirUpdateVendoredFiles'], ...config, }, }); @@ -198,110 +196,50 @@ describe('modules/manager/vendir/artifacts', () => { { file: { type: 'addition', - path: 'vendir.yml', + path: 'vendir.lock.yml', contents: vendirLockFile2, }, }, { file: { type: 'addition', - path: 'charts/example-1.9.2.tgz', + path: 'vendor/Chart.yaml', contents: undefined, }, }, - { - file: { - type: 'deletion', - path: 'charts/example-1.6.2.tgz', - }, - }, - ]); - expect(execSnapshots).toMatchObject([ - { - cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', - }, - { - cmd: "helm dependency update ''", - }, - ]); - }); - - it('add sub chart artifacts to file list if vendir.yml is missing', async () => { - fs.readLocalFile.mockResolvedValueOnce(null); - fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); - const execSnapshots = mockExecAll(); - fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' - ); - fs.getParentDir.mockReturnValue(''); - - // sub chart artifacts - fs.getSiblingFileName.mockReturnValueOnce('charts'); - git.getRepoStatus.mockResolvedValueOnce( - partial({ - not_added: ['charts/example-1.9.2.tgz'], - deleted: ['charts/example-1.6.2.tgz'], - }) - ); - const updatedDeps = [{ depName: 'dep1' }]; - expect( - await vendir.updateArtifacts({ - packageFileName: 'vendir.yml', - updatedDeps, - newPackageFileContent: vendirFile, - config: { - postUpdateOptions: ['helmUpdateSubChartArchives'], - ...config, - }, - }) - ).toEqual([ { file: { type: 'addition', - path: 'charts/example-1.9.2.tgz', + path: 'vendor/my-chart/Chart.yaml', contents: undefined, }, }, { file: { type: 'deletion', - path: 'charts/example-1.6.2.tgz', + path: 'vendor/removed.yaml', }, }, ]); expect(execSnapshots).toMatchObject([ { - cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', + cmd: 'vendir sync', options: { env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', - }, - }, - }, - { - cmd: "helm dependency update ''", - options: { - env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', }, }, }, ]); }); - it('add sub chart artifacts without old archives', async () => { + it('add artifacts to file list if vendir.lock.yml is missing', async () => { fs.readLocalFile.mockResolvedValueOnce(null); fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); const execSnapshots = mockExecAll(); @@ -310,11 +248,12 @@ describe('modules/manager/vendir/artifacts', () => { ); fs.getParentDir.mockReturnValue(''); - // sub chart artifacts - fs.getSiblingFileName.mockReturnValueOnce('charts'); + // artifacts + fs.getSiblingFileName.mockReturnValueOnce('vendor'); git.getRepoStatus.mockResolvedValueOnce( partial({ - not_added: ['charts/example-1.9.2.tgz'], + not_added: ['vendor/Chart.yaml'], + deleted: ['vendor/removed.yaml'], }) ); const updatedDeps = [{ depName: 'dep1' }]; @@ -324,7 +263,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['helmUpdateSubChartArchives'], + postUpdateOptions: ['vendirUpdateVendoredFiles'], ...config, }, }) @@ -332,44 +271,36 @@ describe('modules/manager/vendir/artifacts', () => { { file: { type: 'addition', - path: 'charts/example-1.9.2.tgz', + path: 'vendor/Chart.yaml', contents: undefined, }, }, - ]); - expect(execSnapshots).toMatchObject([ { - cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', - options: { - env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', - }, + file: { + type: 'deletion', + path: 'vendor/removed.yaml', }, }, + ]); + expect(execSnapshots).toMatchObject([ { - cmd: "helm dependency update ''", + cmd: 'vendir sync', options: { env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', }, }, }, ]); }); - it('add sub chart artifacts and ignore files outside of the chart folder', async () => { + it('add artifacts without old archives', async () => { fs.readLocalFile.mockResolvedValueOnce(null); fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); const execSnapshots = mockExecAll(); @@ -378,12 +309,11 @@ describe('modules/manager/vendir/artifacts', () => { ); fs.getParentDir.mockReturnValue(''); - // sub chart artifacts - fs.getSiblingFileName.mockReturnValueOnce('charts'); + // artifacts + fs.getSiblingFileName.mockReturnValueOnce('vendor'); git.getRepoStatus.mockResolvedValueOnce( partial({ - not_added: ['charts/example-1.9.2.tgz', 'exampleFile'], - deleted: ['charts/example-1.6.2.tgz', 'aFolder/otherFile'], + not_added: ['vendor/Chart.yaml'], }) ); const updatedDeps = [{ depName: 'dep1' }]; @@ -393,7 +323,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['helmUpdateSubChartArchives'], + postUpdateOptions: ['vendirUpdateVendoredFiles'], ...config, }, }) @@ -401,103 +331,23 @@ describe('modules/manager/vendir/artifacts', () => { { file: { type: 'addition', - path: 'charts/example-1.9.2.tgz', + path: 'vendor/Chart.yaml', contents: undefined, }, }, - { - file: { - type: 'deletion', - path: 'charts/example-1.6.2.tgz', - }, - }, ]); expect(execSnapshots).toMatchObject([ { - cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', - options: { - env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', - }, - }, - }, - { - cmd: "helm dependency update ''", - options: { - env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', - }, - }, - }, - ]); - }); - - it('skip artifacts which are not lock files or in the chart folder', async () => { - fs.readLocalFile.mockResolvedValueOnce(null); - fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); - const execSnapshots = mockExecAll(); - fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' - ); - fs.getParentDir.mockReturnValue(''); - - // sub chart artifacts - fs.getSiblingFileName.mockReturnValueOnce('charts'); - git.getRepoStatus.mockResolvedValueOnce( - partial({ - modified: ['example/example.tgz'], - }) - ); - const updatedDeps = [{ depName: 'dep1' }]; - expect( - await vendir.updateArtifacts({ - packageFileName: 'vendir.yml', - updatedDeps, - newPackageFileContent: vendirFile, - config: { - postUpdateOptions: ['helmUpdateSubChartArchives'], - ...config, - }, - }) - ).toBeNull(); - expect(execSnapshots).toMatchObject([ - { - cmd: 'helm repo add repo-test https://gitlab.com/api/v4/projects/xxxxxxx/packages/helm/stable', - options: { - env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', - }, - }, - }, - { - cmd: "helm dependency update ''", + cmd: 'vendir sync', options: { env: { - HELM_EXPERIMENTAL_OCI: '1', - HELM_REGISTRY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/registry.json', - HELM_REPOSITORY_CONFIG: - '/tmp/renovate/cache/__renovate-private-cache/repositories.yaml', - HELM_REPOSITORY_CACHE: - '/tmp/renovate/cache/__renovate-private-cache/repositories', + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', }, }, }, diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 21a3990f560a17..941b9961bd941e 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -1,6 +1,7 @@ import is from '@sindresorhus/is'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; +import { coerceArray } from '../../../util/array'; import { exec } from '../../../util/exec'; import type { ExecOptions, ToolConstraint } from '../../../util/exec/types'; import { @@ -51,14 +52,12 @@ export async function updateArtifacts({ try { await writeLocalFile(packageFileName, newPackageFileContent); logger.debug('Updating Vendir artifacts'); - const vendirToolConstraint: ToolConstraint = { - toolName: 'vendir', - constraint: config.constraints?.helm, - }; - const execOptions: ExecOptions = { cwdFile: packageFileName, - toolConstraints: [vendirToolConstraint], + toolConstraints: [ + { toolName: 'vendir', constraint: config.constraints?.vendir }, + { toolName: 'helm', constraint: config.constraints?.helm }, + ], }; await vendirCommands(execOptions); logger.debug('Returning updated Vendir artifacts'); @@ -84,37 +83,25 @@ export async function updateArtifacts({ // add modified vendir archives to artifacts if (is.truthy(isUpdateOptionAddVendirArchives)) { - const vendirPath = getSiblingFileName( - packageFileName, - getParentDir(packageFileName) - ); + // Files must be in the vendor path to get added + const vendorDir = getSiblingFileName(packageFileName, './'); const status = await getRepoStatus(); - const vendirAddition = status.not_added ?? []; - const vendirDeletion = status.deleted ?? []; - - for (const file of vendirAddition) { - // only add artifacts in the vendir sub path - if (!isFileInDir(vendirPath, file)) { - continue; + for (const f of status.modified.concat(status.not_added)) { + if (f.startsWith(vendorDir)) { + fileChanges.push({ + file: { + type: 'addition', + path: f, + contents: await readLocalFile(f), + }, + }); } - fileChanges.push({ - file: { - type: 'addition', - path: file, - contents: await readLocalFile(file), - }, - }); } - - for (const file of vendirDeletion) { - // only add artifacts in the vendir sub path - if (!isFileInDir(vendirPath, file)) { - continue; - } + for (const f of status.deleted ?? []) { fileChanges.push({ file: { type: 'deletion', - path: file, + path: f, }, }); } diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 31b02d4bf14110..efca3ca18948a2 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -1,8 +1,8 @@ import { Fixtures } from '../../../../test/fixtures'; import { extractPackageFile } from '.'; -const oneHelmChart = Fixtures.get('one-helm-chart.yaml'); -const multipleHelmChart = Fixtures.get('multiple-helm-chart.yaml'); +const oneContents = Fixtures.get('one-contents.yaml'); +const multipleContents = Fixtures.get('multiple-contents.yaml'); const emptyDirectories = Fixtures.get('empty-directory.yaml'); describe('modules/manager/vendir/extract', () => { @@ -27,7 +27,7 @@ describe('modules/manager/vendir/extract', () => { }); it('single chart - extracts helm-chart from vendir.yml correctly', () => { - const result = extractPackageFile(oneHelmChart); + const result = extractPackageFile(oneContents); expect(result).toMatchObject({ deps: [ { @@ -41,7 +41,7 @@ describe('modules/manager/vendir/extract', () => { }); it('multiple charts - extracts helm-chart from vendir.yml correctly', () => { - const result = extractPackageFile(multipleHelmChart); + const result = extractPackageFile(multipleContents); expect(result).toMatchObject({ deps: [ { diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index f61e25dd701873..e178ccdd2449ec 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -4,6 +4,7 @@ import { logger } from '../../../logger'; import { HelmDatasource } from '../../datasource/helm'; import type { PackageDependency, PackageFileContent } from '../types'; import type { HelmChart, Vendir } from './types'; +import { isHelmChart } from './utils'; // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. @@ -64,20 +65,17 @@ export function extractPackageFile( // TODO: Add support for OCI Repos by translating Registry URLs and using // Docker datasource. (See Helmv3 for example implementation) const contents = pkg.directories.flatMap((directory) => directory.contents); - const charts = contents.filter( - (chart) => typeof chart.helmChart === 'object' - ); - charts.forEach((chart) => { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - const dep = extractHelmChart(chart.helmChart); - if (dep) { - deps.push({ - ...dep, - depType: 'HelmChart', - }); + for (const content of contents) { + if (isHelmChart(content)) { + const dep = extractHelmChart(content.helmChart); + if (dep) { + deps.push({ + ...dep, + depType: 'HelmChart', + }); + } } - }); + } if (!deps.length) { return null; diff --git a/lib/modules/manager/vendir/types.ts b/lib/modules/manager/vendir/types.ts index fd91cb21ccc712..d80b8aedd1b5cb 100644 --- a/lib/modules/manager/vendir/types.ts +++ b/lib/modules/manager/vendir/types.ts @@ -10,9 +10,15 @@ export interface Directories { contents: Contents[]; } -export interface Contents { +export type Contents = HelmChartContent | OtherContent; + +export interface HelmChartContent { + path: string; + helmChart: HelmChart; +} + +export interface OtherContent { path: string; - helmChart?: HelmChart; } export interface HelmChart { diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index 5930419fcf968b..86868a40e44eb6 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -1,7 +1,12 @@ import upath from 'upath'; +import type { Contents, HelmChartContent } from './types'; export function isFileInDir(dir: string, file: string): boolean { - return upath.dirname(file) === dir; + return upath.relative(dir, file).startsWith('./'); +} + +export function isHelmChart(item: Contents): item is HelmChartContent { + return 'helmChart' in item; } // TODO: Add Support for Registry Aliases (See Helmv3 for possible implementation) diff --git a/lib/util/exec/containerbase.ts b/lib/util/exec/containerbase.ts index e8528aaadca01f..fbc7f2c576f58f 100644 --- a/lib/util/exec/containerbase.ts +++ b/lib/util/exec/containerbase.ts @@ -196,6 +196,11 @@ const allToolConfig: Record = { packageName: 'flutter', versioning: semverVersioningId, }, + vendir: { + datasource: 'github-releases', + packageName: 'carvel-dev/vendir', + versioning: semverVersioningId, + }, }; export function supportsDynamicInstall(toolName: string): boolean { From 068951f79956d32a657af2d489511d5a4e58a129 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 7 Feb 2024 07:35:30 -0500 Subject: [PATCH 04/52] fix: swap toMatchObject to toEqual chore: revert package.json change chore: run prettier --- lib/modules/manager/vendir/README.md | 4 ++-- lib/modules/manager/vendir/artifacts.spec.ts | 4 ++-- lib/modules/manager/vendir/artifacts.ts | 5 +---- package.json | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/modules/manager/vendir/README.md b/lib/modules/manager/vendir/README.md index 87ce863961ec58..da0173d378d35e 100644 --- a/lib/modules/manager/vendir/README.md +++ b/lib/modules/manager/vendir/README.md @@ -6,7 +6,7 @@ helmChart: # chart name (required) name: stable/redis # use specific chart version (string; optional) - version: "1.2.1" + version: '1.2.1' # specifies Helm repository to fetch from (optional) repository: # repository url; supports exprimental oci helm fetch via @@ -23,5 +23,5 @@ helmChart: name: my-helm-auth # specify helm binary version to use; # '3' means binary 'helm3' needs to be on the path (optional) - helmVersion: "3" + helmVersion: '3' ``` diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 765d095eca7fe5..2942bdcd4ab505 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -125,7 +125,7 @@ describe('modules/manager/vendir/artifacts', () => { newPackageFileContent: vendirFile, config: { ...config, updateType: 'lockFileMaintenance' }, }) - ).toMatchObject([ + ).toEqual([ { file: { type: 'addition', @@ -154,7 +154,7 @@ describe('modules/manager/vendir/artifacts', () => { newPackageFileContent: vendirFile, config, }) - ).toMatchObject([ + ).toEqual([ { artifactError: { lockFile: 'vendir.yml', diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 941b9961bd941e..66b34c9776d3ea 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -1,18 +1,15 @@ import is from '@sindresorhus/is'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; -import { coerceArray } from '../../../util/array'; import { exec } from '../../../util/exec'; -import type { ExecOptions, ToolConstraint } from '../../../util/exec/types'; +import type { ExecOptions } from '../../../util/exec/types'; import { - getParentDir, getSiblingFileName, readLocalFile, writeLocalFile, } from '../../../util/fs'; import { getRepoStatus } from '../../../util/git'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; -import { isFileInDir } from './utils'; async function vendirCommands(execOptions: ExecOptions): Promise { const cmd: string[] = []; diff --git a/package.json b/package.json index bee23cc7fc5788..65cbbec7e09ae1 100644 --- a/package.json +++ b/package.json @@ -143,12 +143,12 @@ "node": "^18.12.0 || >=20.0.0" }, "dependencies": { + "@aws-sdk/credential-providers": "3.363.0", "@aws-sdk/client-codecommit": "3.363.0", "@aws-sdk/client-ec2": "3.363.0", "@aws-sdk/client-ecr": "3.363.0", "@aws-sdk/client-rds": "3.363.0", "@aws-sdk/client-s3": "3.363.0", - "@aws-sdk/credential-providers": "3.363.0", "@breejs/later": "4.1.0", "@cdktf/hcl2json": "0.18.0", "@iarna/toml": "3.0.0", From d68c8e377ae86edfdc72c28f92474fac69419279 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 7 Feb 2024 07:40:46 -0500 Subject: [PATCH 05/52] chore: run prettier --- lib/modules/manager/vendir/artifacts.spec.ts | 36 ++++++++++---------- lib/modules/manager/vendir/artifacts.ts | 2 +- lib/modules/manager/vendir/extract.ts | 6 ++-- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 2942bdcd4ab505..32ac92f60d18fd 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -44,7 +44,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: '', config, - }) + }), ).toBeNull(); }); @@ -55,7 +55,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps: [], newPackageFileContent: '', config, - }) + }), ).toBeNull(); }); @@ -65,7 +65,7 @@ describe('modules/manager/vendir/artifacts', () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); const updatedDeps = [{ depName: 'dep1' }]; @@ -75,7 +75,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config, - }) + }), ).toBeNull(); expect(execSnapshots).toMatchSnapshot(); }); @@ -86,7 +86,7 @@ describe('modules/manager/vendir/artifacts', () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); const updatedDeps = [{ depName: 'dep1' }]; @@ -96,7 +96,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config, - }) + }), ).toEqual([ { file: { @@ -115,7 +115,7 @@ describe('modules/manager/vendir/artifacts', () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); expect( @@ -124,7 +124,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps: [], newPackageFileContent: vendirFile, config: { ...config, updateType: 'lockFileMaintenance' }, - }) + }), ).toEqual([ { file: { @@ -141,7 +141,7 @@ describe('modules/manager/vendir/artifacts', () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.writeLocalFile.mockImplementationOnce(() => { throw new Error('not found'); @@ -153,7 +153,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config, - }) + }), ).toEqual([ { artifactError: { @@ -170,7 +170,7 @@ describe('modules/manager/vendir/artifacts', () => { const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); @@ -180,7 +180,7 @@ describe('modules/manager/vendir/artifacts', () => { partial({ not_added: ['vendor/Chart.yaml', 'vendor/my-chart/Chart.yaml'], deleted: ['vendor/removed.yaml'], - }) + }), ); const updatedDeps = [{ depName: 'dep1' }]; const test = await vendir.updateArtifacts({ @@ -244,7 +244,7 @@ describe('modules/manager/vendir/artifacts', () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); @@ -254,7 +254,7 @@ describe('modules/manager/vendir/artifacts', () => { partial({ not_added: ['vendor/Chart.yaml'], deleted: ['vendor/removed.yaml'], - }) + }), ); const updatedDeps = [{ depName: 'dep1' }]; expect( @@ -266,7 +266,7 @@ describe('modules/manager/vendir/artifacts', () => { postUpdateOptions: ['vendirUpdateVendoredFiles'], ...config, }, - }) + }), ).toEqual([ { file: { @@ -305,7 +305,7 @@ describe('modules/manager/vendir/artifacts', () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache' + '/tmp/renovate/cache/__renovate-private-cache', ); fs.getParentDir.mockReturnValue(''); @@ -314,7 +314,7 @@ describe('modules/manager/vendir/artifacts', () => { git.getRepoStatus.mockResolvedValueOnce( partial({ not_added: ['vendor/Chart.yaml'], - }) + }), ); const updatedDeps = [{ depName: 'dep1' }]; expect( @@ -326,7 +326,7 @@ describe('modules/manager/vendir/artifacts', () => { postUpdateOptions: ['vendirUpdateVendoredFiles'], ...config, }, - }) + }), ).toEqual([ { file: { diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 66b34c9776d3ea..3e62042a4273df 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -29,7 +29,7 @@ export async function updateArtifacts({ const isLockFileMaintenance = config.updateType === 'lockFileMaintenance'; const isUpdateOptionAddVendirArchives = config.postUpdateOptions?.includes( - 'vendirUpdateSubChartArchives' + 'vendirUpdateSubChartArchives', ); if ( diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index e178ccdd2449ec..649ec08d6f931e 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -10,7 +10,7 @@ import { isHelmChart } from './utils'; // Recommend looking at the kustomize manager for more information on support. export function extractHelmChart( - helmChart: HelmChart + helmChart: HelmChart, ): PackageDependency | null { if (!helmChart.name) { return null; @@ -26,7 +26,7 @@ export function extractHelmChart( export function parseVendir( content: string, - packageFile?: string + packageFile?: string, ): Vendir | null { let pkg: Vendir | null = null; try { @@ -51,7 +51,7 @@ export function parseVendir( export function extractPackageFile( content: string, - packageFile?: string // TODO: fix tests + packageFile?: string, // TODO: fix tests ): PackageFileContent | null { logger.trace(`vendir.extractPackageFile(${packageFile!})`); const deps: PackageDependency[] = []; From 10b9154454e128f6385be1041e3cf529e12c0411 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 7 Feb 2024 09:36:41 -0500 Subject: [PATCH 06/52] fix: use yaml lib fix: inline function fix: remove resetAllMocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Zoltán Reegn --- lib/modules/manager/vendir/artifacts.spec.ts | 20 ++++++++++---------- lib/modules/manager/vendir/artifacts.ts | 12 +++--------- lib/modules/manager/vendir/extract.spec.ts | 3 --- lib/modules/manager/vendir/extract.ts | 4 ++-- 4 files changed, 15 insertions(+), 24 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 32ac92f60d18fd..0aaaf10c4e9293 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -1,6 +1,6 @@ import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; -import { envMock, mockExecAll } from '../../../../test/exec-util'; +import { envMock, mockExecAll, mockExecSequence } from '../../../../test/exec-util'; import { Fixtures } from '../../../../test/fixtures'; import { env, fs, git, partial } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; @@ -60,10 +60,10 @@ describe('modules/manager/vendir/artifacts', () => { }); it('returns null if unchanged', async () => { - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); @@ -77,7 +77,7 @@ describe('modules/manager/vendir/artifacts', () => { config, }), ).toBeNull(); - expect(execSnapshots).toMatchSnapshot(); + expect(execSnapshots).toMatchSnapshot([{ cmd: 'vendir sync' }]); }); it('returns updated vendir.lock', async () => { @@ -110,10 +110,10 @@ describe('modules/manager/vendir/artifacts', () => { }); it('returns updated vendir.yml for lockfile maintenance', async () => { - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); @@ -139,7 +139,7 @@ describe('modules/manager/vendir/artifacts', () => { it('catches errors', async () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as any); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); @@ -188,7 +188,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateVendoredFiles'], + postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }); @@ -263,7 +263,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateVendoredFiles'], + postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }), @@ -323,7 +323,7 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateVendoredFiles'], + postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }), diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 3e62042a4273df..b028ed0e968237 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -11,14 +11,6 @@ import { import { getRepoStatus } from '../../../util/git'; import type { UpdateArtifact, UpdateArtifactsResult } from '../types'; -async function vendirCommands(execOptions: ExecOptions): Promise { - const cmd: string[] = []; - - cmd.push(`vendir sync`); - - await exec(cmd, execOptions); -} - export async function updateArtifacts({ packageFileName, updatedDeps, @@ -56,7 +48,9 @@ export async function updateArtifacts({ { toolName: 'helm', constraint: config.constraints?.helm }, ], }; - await vendirCommands(execOptions); + + await exec(`vendir sync`, execOptions); + logger.debug('Returning updated Vendir artifacts'); const fileChanges: UpdateArtifactsResult[] = []; diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index efca3ca18948a2..14e25194cb0ced 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -7,9 +7,6 @@ const emptyDirectories = Fixtures.get('empty-directory.yaml'); describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { - beforeEach(() => { - jest.resetAllMocks(); - }); it('returns null for invalid yaml file content', () => { const result = extractPackageFile('nothing here: ['); diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 649ec08d6f931e..cc7f25019560b4 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -1,5 +1,5 @@ import is from '@sindresorhus/is'; -import { load } from 'js-yaml'; +import { parseSingleYaml } from '../../../util/yaml'; import { logger } from '../../../logger'; import { HelmDatasource } from '../../datasource/helm'; import type { PackageDependency, PackageFileContent } from '../types'; @@ -30,7 +30,7 @@ export function parseVendir( ): Vendir | null { let pkg: Vendir | null = null; try { - pkg = load(content, { json: true }) as Vendir; + pkg = parseSingleYaml(content); } catch (e) /* istanbul ignore next */ { logger.debug({ packageFile }, 'Error parsing vendir.yml file'); return null; From bb505392dd0ef3becd68f22b6b720284a18a2898 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 7 Feb 2024 09:43:16 -0500 Subject: [PATCH 07/52] chore: remove extra import --- lib/modules/manager/vendir/artifacts.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 0aaaf10c4e9293..309f494914682d 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -1,6 +1,6 @@ import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; -import { envMock, mockExecAll, mockExecSequence } from '../../../../test/exec-util'; +import { envMock, mockExecAll } from '../../../../test/exec-util'; import { Fixtures } from '../../../../test/fixtures'; import { env, fs, git, partial } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; From 91d52e810a1d447c1e2ec19f20344885664d4523 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Mon, 4 Mar 2024 10:35:26 +0100 Subject: [PATCH 08/52] Update lib/modules/manager/vendir/README.md Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- lib/modules/manager/vendir/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/README.md b/lib/modules/manager/vendir/README.md index da0173d378d35e..411790b678ba25 100644 --- a/lib/modules/manager/vendir/README.md +++ b/lib/modules/manager/vendir/README.md @@ -1,4 +1,4 @@ -The [vendir](https://carvel.dev/vendir/) manager only updates dependencies of the `helmChart` type. +The [vendir](https://carvel.dev/vendir/) manager can only update `helmChart`-type dependencies. ```yaml # fetch Helm chart contents (optional; v0.11.0+) From e773e297f8a305947e8c62c96cfc8db059806274 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 8 Mar 2024 12:22:45 +0100 Subject: [PATCH 09/52] Update lib/modules/manager/vendir/README.md Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com> --- lib/modules/manager/vendir/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/README.md b/lib/modules/manager/vendir/README.md index 411790b678ba25..6e9c5d47e641df 100644 --- a/lib/modules/manager/vendir/README.md +++ b/lib/modules/manager/vendir/README.md @@ -1,6 +1,6 @@ The [vendir](https://carvel.dev/vendir/) manager can only update `helmChart`-type dependencies. -```yaml +```yaml title="Example helmChart" # fetch Helm chart contents (optional; v0.11.0+) helmChart: # chart name (required) From 3113d9519014f8ebdd458b8b79adeb4ac2e8252a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 14 Mar 2024 15:35:11 +0100 Subject: [PATCH 10/52] fix failing tests --- lib/modules/manager/vendir/artifacts.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index b028ed0e968237..71ba747a1adb3d 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -77,7 +77,7 @@ export async function updateArtifacts({ // Files must be in the vendor path to get added const vendorDir = getSiblingFileName(packageFileName, './'); const status = await getRepoStatus(); - for (const f of status.modified.concat(status.not_added)) { + for (const f of (status.modified ?? []).concat(status.not_added)) { if (f.startsWith(vendorDir)) { fileChanges.push({ file: { @@ -88,6 +88,7 @@ export async function updateArtifacts({ }); } } + for (const f of status.deleted ?? []) { fileChanges.push({ file: { From a229cea0e334ca54e32ae4895bfb97cfb2f7bed3 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 14 Mar 2024 15:42:10 +0100 Subject: [PATCH 11/52] fix: linters --- lib/modules/manager/vendir/extract.spec.ts | 1 - lib/modules/manager/vendir/extract.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 14e25194cb0ced..ba4c38be24df13 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -7,7 +7,6 @@ const emptyDirectories = Fixtures.get('empty-directory.yaml'); describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { - it('returns null for invalid yaml file content', () => { const result = extractPackageFile('nothing here: ['); expect(result).toBeNull(); diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index cc7f25019560b4..53fb39c03179c1 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; -import { parseSingleYaml } from '../../../util/yaml'; import { logger } from '../../../logger'; +import { parseSingleYaml } from '../../../util/yaml'; import { HelmDatasource } from '../../datasource/helm'; import type { PackageDependency, PackageFileContent } from '../types'; import type { HelmChart, Vendir } from './types'; From 75ddfb67c8e0a72cb34851f01aecde0a1937f760 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 14 Mar 2024 15:45:49 +0100 Subject: [PATCH 12/52] Fix lint --- lib/modules/manager/vendir/{README.md => readme.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/modules/manager/vendir/{README.md => readme.md} (100%) diff --git a/lib/modules/manager/vendir/README.md b/lib/modules/manager/vendir/readme.md similarity index 100% rename from lib/modules/manager/vendir/README.md rename to lib/modules/manager/vendir/readme.md From 0791bd5739e4f01599cf2dd36b94267231e14359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 14 Mar 2024 16:24:14 +0100 Subject: [PATCH 13/52] Add helm OCI registry support --- .../vendir/__fixtures__/oci-contents.yaml | 12 ++++++++++++ lib/modules/manager/vendir/extract.spec.ts | 15 +++++++++++++++ lib/modules/manager/vendir/extract.ts | 18 +++++++++++------- lib/modules/manager/vendir/utils.ts | 12 +++++++++++- 4 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 lib/modules/manager/vendir/__fixtures__/oci-contents.yaml diff --git a/lib/modules/manager/vendir/__fixtures__/oci-contents.yaml b/lib/modules/manager/vendir/__fixtures__/oci-contents.yaml new file mode 100644 index 00000000000000..d34ad87fb52c64 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/oci-contents.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: custom-repo-custom-version + helmChart: + name: contour + version: "7.10.1" + repository: + url: oci://charts.bitnami.com/bitnami diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index ba4c38be24df13..4dd348eb87d38a 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -2,6 +2,7 @@ import { Fixtures } from '../../../../test/fixtures'; import { extractPackageFile } from '.'; const oneContents = Fixtures.get('one-contents.yaml'); +const ociContents = Fixtures.get('oci-contents.yaml'); const multipleContents = Fixtures.get('multiple-contents.yaml'); const emptyDirectories = Fixtures.get('empty-directory.yaml'); @@ -36,6 +37,20 @@ describe('modules/manager/vendir/extract', () => { }); }); + it('single chart - extracts oci helm-chart from vendir.yml correctly', () => { + const result = extractPackageFile(ociContents); + expect(result).toMatchObject({ + deps: [ + { + currentValue: '7.10.1', + depName: 'contour', + datasource: 'docker', + registryUrls: ['https://charts.bitnami.com/bitnami'], + }, + ], + }); + }); + it('multiple charts - extracts helm-chart from vendir.yml correctly', () => { const result = extractPackageFile(multipleContents); expect(result).toMatchObject({ diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 53fb39c03179c1..22560fb1a5bd71 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -4,7 +4,8 @@ import { parseSingleYaml } from '../../../util/yaml'; import { HelmDatasource } from '../../datasource/helm'; import type { PackageDependency, PackageFileContent } from '../types'; import type { HelmChart, Vendir } from './types'; -import { isHelmChart } from './utils'; +import { isHelmChart, isOCIRegistry } from './utils'; +import { DockerDatasource } from '../../datasource/docker'; // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. @@ -15,12 +16,17 @@ export function extractHelmChart( if (!helmChart.name) { return null; } - + let registryUrl = helmChart.repository.url + let dataSource = HelmDatasource.id + if (isOCIRegistry(helmChart.repository)) { + registryUrl = helmChart.repository.url.replace("oci://", "https://") + dataSource = DockerDatasource.id + } return { depName: helmChart.name, currentValue: helmChart.version, - registryUrls: [helmChart.repository.url], - datasource: HelmDatasource.id, + registryUrls: [registryUrl], + datasource: dataSource, }; } @@ -51,7 +57,7 @@ export function parseVendir( export function extractPackageFile( content: string, - packageFile?: string, // TODO: fix tests + packageFile?: string, ): PackageFileContent | null { logger.trace(`vendir.extractPackageFile(${packageFile!})`); const deps: PackageDependency[] = []; @@ -62,8 +68,6 @@ export function extractPackageFile( } // grab the helm charts - // TODO: Add support for OCI Repos by translating Registry URLs and using - // Docker datasource. (See Helmv3 for example implementation) const contents = pkg.directories.flatMap((directory) => directory.contents); for (const content of contents) { if (isHelmChart(content)) { diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index 86868a40e44eb6..6ace7e86e37833 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -1,5 +1,6 @@ import upath from 'upath'; -import type { Contents, HelmChartContent } from './types'; +import type { Contents, HelmChartContent, Repository } from './types'; +import is from '@sindresorhus/is'; export function isFileInDir(dir: string, file: string): boolean { return upath.relative(dir, file).startsWith('./'); @@ -9,4 +10,13 @@ export function isHelmChart(item: Contents): item is HelmChartContent { return 'helmChart' in item; } +export function isOCIRegistry( + repository: Repository, +): boolean { + if (is.nullOrUndefined(repository)) { + return false; + } + return repository.url.startsWith('oci://'); +} + // TODO: Add Support for Registry Aliases (See Helmv3 for possible implementation) From 9ac3cc7d2bb73d80de37fe1c9501d6d2d9e05cb1 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 14 Mar 2024 16:35:22 +0100 Subject: [PATCH 14/52] chore: lint fix --- lib/modules/manager/vendir/extract.ts | 10 +++++----- lib/modules/manager/vendir/utils.ts | 6 ++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 22560fb1a5bd71..ab6de089dcad46 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -1,11 +1,11 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; import { parseSingleYaml } from '../../../util/yaml'; +import { DockerDatasource } from '../../datasource/docker'; import { HelmDatasource } from '../../datasource/helm'; import type { PackageDependency, PackageFileContent } from '../types'; import type { HelmChart, Vendir } from './types'; import { isHelmChart, isOCIRegistry } from './utils'; -import { DockerDatasource } from '../../datasource/docker'; // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. @@ -16,11 +16,11 @@ export function extractHelmChart( if (!helmChart.name) { return null; } - let registryUrl = helmChart.repository.url - let dataSource = HelmDatasource.id + let registryUrl = helmChart.repository.url; + let dataSource = HelmDatasource.id; if (isOCIRegistry(helmChart.repository)) { - registryUrl = helmChart.repository.url.replace("oci://", "https://") - dataSource = DockerDatasource.id + registryUrl = helmChart.repository.url.replace('oci://', 'https://'); + dataSource = DockerDatasource.id; } return { depName: helmChart.name, diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index 6ace7e86e37833..ea3d4adcf9dca6 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -1,6 +1,6 @@ +import is from '@sindresorhus/is'; import upath from 'upath'; import type { Contents, HelmChartContent, Repository } from './types'; -import is from '@sindresorhus/is'; export function isFileInDir(dir: string, file: string): boolean { return upath.relative(dir, file).startsWith('./'); @@ -10,9 +10,7 @@ export function isHelmChart(item: Contents): item is HelmChartContent { return 'helmChart' in item; } -export function isOCIRegistry( - repository: Repository, -): boolean { +export function isOCIRegistry(repository: Repository): boolean { if (is.nullOrUndefined(repository)) { return false; } From 1377c979669ae6b2b7f9ad2addce406eccf2e7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 14 Mar 2024 16:42:02 +0100 Subject: [PATCH 15/52] Drop unnecessary code paths --- lib/modules/manager/vendir/extract.ts | 10 ---------- lib/modules/manager/vendir/utils.ts | 11 ----------- 2 files changed, 21 deletions(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index ab6de089dcad46..dadb245eb7df06 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -13,9 +13,6 @@ import { isHelmChart, isOCIRegistry } from './utils'; export function extractHelmChart( helmChart: HelmChart, ): PackageDependency | null { - if (!helmChart.name) { - return null; - } let registryUrl = helmChart.repository.url; let dataSource = HelmDatasource.id; if (isOCIRegistry(helmChart.repository)) { @@ -45,13 +42,6 @@ export function parseVendir( if (!pkg || is.string(pkg)) { return null; } - - pkg.kind ??= 'Config'; - - if (!['Config'].includes(pkg.kind)) { - return null; - } - return pkg; } diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index ea3d4adcf9dca6..c9ec6eb6f2347a 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -1,20 +1,9 @@ -import is from '@sindresorhus/is'; -import upath from 'upath'; import type { Contents, HelmChartContent, Repository } from './types'; -export function isFileInDir(dir: string, file: string): boolean { - return upath.relative(dir, file).startsWith('./'); -} - export function isHelmChart(item: Contents): item is HelmChartContent { return 'helmChart' in item; } export function isOCIRegistry(repository: Repository): boolean { - if (is.nullOrUndefined(repository)) { - return false; - } return repository.url.startsWith('oci://'); } - -// TODO: Add Support for Registry Aliases (See Helmv3 for possible implementation) From 42c7ae4bb0e446482169d012cae82baeb3e37e3d Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 14 Mar 2024 19:49:12 +0100 Subject: [PATCH 16/52] fix: Add files correctly fix: add back conditional for vendirSync properly docs: update vendir readme --- lib/modules/manager/vendir/artifacts.ts | 9 +++++++-- lib/modules/manager/vendir/readme.md | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 71ba747a1adb3d..d68baa078b7a63 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -74,11 +74,16 @@ export async function updateArtifacts({ // add modified vendir archives to artifacts if (is.truthy(isUpdateOptionAddVendirArchives)) { + logger.debug("Adding Sync'd files to git"); // Files must be in the vendor path to get added - const vendorDir = getSiblingFileName(packageFileName, './'); + const vendorDir = getSiblingFileName(packageFileName, '.'); + logger.debug('vendorDir = ' + vendorDir); const status = await getRepoStatus(); for (const f of (status.modified ?? []).concat(status.not_added)) { - if (f.startsWith(vendorDir)) { + logger.debug({ f }, 'Checking if file is in vendor directory'); + const isFileInVendorDir = f.startsWith(vendorDir); + if (vendorDir === '.' || isFileInVendorDir) { + logger.debug({ f }, 'Adding file to artifacts/git'); fileChanges.push({ file: { type: 'addition', diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 6e9c5d47e641df..d4402cb194f6a1 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -1,4 +1,7 @@ -The [vendir](https://carvel.dev/vendir/) manager can only update `helmChart`-type dependencies. + +Renovate supports updating Helm Chart references in vendir.yml via the [vendir](https://carvel.dev/vendir/) tool + +It supports both https and oci helm chart repositories. ```yaml title="Example helmChart" # fetch Helm chart contents (optional; v0.11.0+) @@ -25,3 +28,14 @@ helmChart: # '3' means binary 'helm3' needs to be on the path (optional) helmVersion: '3' ``` + +### Subchart Archives + +To get updates for subchart archives put `vendirUpdateSubChartArchives` in your `postUpdateOptions` configuration. +Renovate now updates archives in the folder specified by your vendir.yml. + +```json +{ + "postUpdateOptions": ["vendirUpdateSubChartArchives"] +} +``` From 47ecf1ee44e7093f713f61fdd1886ad971c25a6c Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 15 Mar 2024 18:16:37 +0100 Subject: [PATCH 17/52] feat: passing tests --- lib/modules/manager/vendir/artifacts.spec.ts | 11 ++++------- lib/modules/manager/vendir/artifacts.ts | 5 +++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 309f494914682d..d81b8efb600ed9 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -81,10 +81,10 @@ describe('modules/manager/vendir/artifacts', () => { }); it('returns updated vendir.lock', async () => { - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); @@ -165,10 +165,10 @@ describe('modules/manager/vendir/artifacts', () => { }); it('add artifacts to file list if vendir.yml exists', async () => { - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); const execSnapshots = mockExecAll(); - fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2 as never); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); @@ -188,7 +188,6 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }); @@ -263,7 +262,6 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }), @@ -323,7 +321,6 @@ describe('modules/manager/vendir/artifacts', () => { updatedDeps, newPackageFileContent: vendirFile, config: { - postUpdateOptions: ['vendirUpdateSubChartArchives'], ...config, }, }), diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index d68baa078b7a63..7fef2312a21597 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -4,6 +4,7 @@ import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; import type { ExecOptions } from '../../../util/exec/types'; import { + getParentDir, getSiblingFileName, readLocalFile, writeLocalFile, @@ -73,10 +74,10 @@ export async function updateArtifacts({ } // add modified vendir archives to artifacts - if (is.truthy(isUpdateOptionAddVendirArchives)) { + if (isUpdateOptionAddVendirArchives === true) { logger.debug("Adding Sync'd files to git"); // Files must be in the vendor path to get added - const vendorDir = getSiblingFileName(packageFileName, '.'); + const vendorDir = getParentDir(packageFileName); logger.debug('vendorDir = ' + vendorDir); const status = await getRepoStatus(); for (const f of (status.modified ?? []).concat(status.not_added)) { From 49ad5c7198ddee317c11db4644969aab99e5f6bd Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 15 Mar 2024 19:06:08 +0100 Subject: [PATCH 18/52] fix: pr comments --- .../vendir/__fixtures__/empty-directory.yaml | 3 - lib/modules/manager/vendir/artifacts.spec.ts | 13 ++- lib/modules/manager/vendir/artifacts.ts | 86 ++++++++----------- lib/modules/manager/vendir/extract.spec.ts | 5 +- 4 files changed, 43 insertions(+), 64 deletions(-) delete mode 100644 lib/modules/manager/vendir/__fixtures__/empty-directory.yaml diff --git a/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml b/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml deleted file mode 100644 index b9e824668b4657..00000000000000 --- a/lib/modules/manager/vendir/__fixtures__/empty-directory.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: vendir.k14s.io/v1alpha1 -kind: Config -directories: [] diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index d81b8efb600ed9..9492f0e307f275 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -10,10 +10,10 @@ import type { UpdateArtifactsConfig } from '../types'; import * as vendir from '.'; jest.mock('../../datasource', () => mockDeep()); -jest.mock('../../../util/exec/env'); -jest.mock('../../../util/http'); -jest.mock('../../../util/fs'); -jest.mock('../../../util/git'); +jest.mock('../../../util/exec/env', () => mockDeep()); +jest.mock('../../../util/http', () => mockDeep()); +jest.mock('../../../util/fs', () => mockDeep()); +jest.mock('../../../util/git', () => mockDeep()); const adminConfig: RepoGlobalConfig = { localDir: join('/tmp/github/some/repo'), // `join` fixes Windows CI @@ -32,10 +32,6 @@ describe('modules/manager/vendir/artifacts', () => { GlobalConfig.set(adminConfig); }); - afterEach(() => { - GlobalConfig.reset(); - }); - it('returns null if no vendir.lock.yml found', async () => { const updatedDeps = [{ depName: 'dep1' }]; expect( @@ -61,6 +57,7 @@ describe('modules/manager/vendir/artifacts', () => { it('returns null if unchanged', async () => { fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce(vendirFile); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); const execSnapshots = mockExecAll(); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 7fef2312a21597..abab5a4df7c6b1 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -1,4 +1,3 @@ -import is from '@sindresorhus/is'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { logger } from '../../../logger'; import { exec } from '../../../util/exec'; @@ -20,22 +19,13 @@ export async function updateArtifacts({ }: UpdateArtifact): Promise { logger.debug(`vendir.updateArtifacts(${packageFileName})`); - const isLockFileMaintenance = config.updateType === 'lockFileMaintenance'; - const isUpdateOptionAddVendirArchives = config.postUpdateOptions?.includes( - 'vendirUpdateSubChartArchives', - ); - - if ( - !isLockFileMaintenance && - (updatedDeps === undefined || updatedDeps.length < 1) - ) { - logger.debug('No updated vendir deps - returning null'); + const lockFileName = getSiblingFileName(packageFileName, 'vendir.lock.yml'); + if (!lockFileName) { + logger.debug('No vendir.lock.yml found'); return null; } - - const lockFileName = getSiblingFileName(packageFileName, 'vendir.lock.yml'); const existingLockFileContent = await readLocalFile(lockFileName, 'utf8'); - if (!existingLockFileContent && !isUpdateOptionAddVendirArchives) { + if (!existingLockFileContent) { logger.debug('No vendir.lock.yml found'); return null; } @@ -56,55 +46,47 @@ export async function updateArtifacts({ const fileChanges: UpdateArtifactsResult[] = []; - if (is.truthy(existingLockFileContent)) { - const newVendirLockContent = await readLocalFile(lockFileName, 'utf8'); - const isLockFileChanged = - existingLockFileContent !== newVendirLockContent; - if (isLockFileChanged) { - fileChanges.push({ - file: { - type: 'addition', - path: lockFileName, - contents: newVendirLockContent, - }, - }); - } else { - logger.debug('vendir.lock.yml is unchanged'); - } + const newVendirLockContent = await readLocalFile(lockFileName, 'utf8'); + const isLockFileChanged = existingLockFileContent !== newVendirLockContent; + if (isLockFileChanged) { + fileChanges.push({ + file: { + type: 'addition', + path: lockFileName, + contents: newVendirLockContent, + }, + }); + } else { + logger.debug('vendir.lock.yml is unchanged'); } // add modified vendir archives to artifacts - if (isUpdateOptionAddVendirArchives === true) { - logger.debug("Adding Sync'd files to git"); - // Files must be in the vendor path to get added - const vendorDir = getParentDir(packageFileName); - logger.debug('vendorDir = ' + vendorDir); - const status = await getRepoStatus(); - for (const f of (status.modified ?? []).concat(status.not_added)) { - logger.debug({ f }, 'Checking if file is in vendor directory'); - const isFileInVendorDir = f.startsWith(vendorDir); - if (vendorDir === '.' || isFileInVendorDir) { - logger.debug({ f }, 'Adding file to artifacts/git'); - fileChanges.push({ - file: { - type: 'addition', - path: f, - contents: await readLocalFile(f), - }, - }); - } - } - - for (const f of status.deleted ?? []) { + logger.debug("Adding Sync'd files to git"); + // Files must be in the vendor path to get added + const vendorDir = getParentDir(packageFileName); + const status = await getRepoStatus(); + for (const f of (status.modified ?? []).concat(status.not_added)) { + const isFileInVendorDir = f.startsWith(vendorDir); + if (vendorDir || isFileInVendorDir) { fileChanges.push({ file: { - type: 'deletion', + type: 'addition', path: f, + contents: await readLocalFile(f), }, }); } } + for (const f of status.deleted ?? []) { + fileChanges.push({ + file: { + type: 'deletion', + path: f, + }, + }); + } + return fileChanges.length > 0 ? fileChanges : null; } catch (err) { // istanbul ignore if diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 4dd348eb87d38a..e41a97b767e582 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -4,7 +4,10 @@ import { extractPackageFile } from '.'; const oneContents = Fixtures.get('one-contents.yaml'); const ociContents = Fixtures.get('oci-contents.yaml'); const multipleContents = Fixtures.get('multiple-contents.yaml'); -const emptyDirectories = Fixtures.get('empty-directory.yaml'); +const emptyDirectories = Fixtures.mock( + { apiVersion: 'vendir.k14s.io/v1alpha1', kind: 'Config', directories: [] }, + 'vendir.yml', +); describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { From ecd9ce588feb3f3b57b3ef78ce24e967261a946b Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 15 Mar 2024 17:07:22 +0100 Subject: [PATCH 19/52] fix: check for vendir.locak.yml existence first --- lib/modules/manager/vendir/artifacts.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index abab5a4df7c6b1..94cf79018945cc 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -5,6 +5,7 @@ import type { ExecOptions } from '../../../util/exec/types'; import { getParentDir, getSiblingFileName, + localPathIsFile, readLocalFile, writeLocalFile, } from '../../../util/fs'; From 864efe7b1951d4e7a2f5bddefc59ecbea81ef8c9 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 15 Mar 2024 21:12:14 +0100 Subject: [PATCH 20/52] feat: return null when no vendir.lock.yml feat: handle status of repo properly tests: remove incorrect test when lockfile didn't exist tests: add lockfile file supposed to be tracked docs: add requirement for lockfile to exist --- lib/modules/manager/vendir/artifacts.spec.ts | 64 ++------------------ lib/modules/manager/vendir/artifacts.ts | 38 +++++++----- lib/modules/manager/vendir/readme.md | 14 +---- 3 files changed, 30 insertions(+), 86 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 9492f0e307f275..cc985d261ffede 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -235,9 +235,10 @@ describe('modules/manager/vendir/artifacts', () => { ]); }); - it('add artifacts to file list if vendir.lock.yml is missing', async () => { - fs.readLocalFile.mockResolvedValueOnce(null); - fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + it('add artifacts without old archives', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', @@ -249,7 +250,6 @@ describe('modules/manager/vendir/artifacts', () => { git.getRepoStatus.mockResolvedValueOnce( partial({ not_added: ['vendor/Chart.yaml'], - deleted: ['vendor/removed.yaml'], }), ); const updatedDeps = [{ depName: 'dep1' }]; @@ -266,62 +266,10 @@ describe('modules/manager/vendir/artifacts', () => { { file: { type: 'addition', - path: 'vendor/Chart.yaml', - contents: undefined, - }, - }, - { - file: { - type: 'deletion', - path: 'vendor/removed.yaml', - }, - }, - ]); - expect(execSnapshots).toMatchObject([ - { - cmd: 'vendir sync', - options: { - env: { - HOME: '/home/user', - HTTPS_PROXY: 'https://example.com', - HTTP_PROXY: 'http://example.com', - LANG: 'en_US.UTF-8', - LC_ALL: 'en_US', - NO_PROXY: 'localhost', - PATH: '/tmp/path', - }, + path: 'vendir.lock.yml', + contents: vendirLockFile2, }, }, - ]); - }); - - it('add artifacts without old archives', async () => { - fs.readLocalFile.mockResolvedValueOnce(null); - fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); - const execSnapshots = mockExecAll(); - fs.privateCacheDir.mockReturnValue( - '/tmp/renovate/cache/__renovate-private-cache', - ); - fs.getParentDir.mockReturnValue(''); - - // artifacts - fs.getSiblingFileName.mockReturnValueOnce('vendor'); - git.getRepoStatus.mockResolvedValueOnce( - partial({ - not_added: ['vendor/Chart.yaml'], - }), - ); - const updatedDeps = [{ depName: 'dep1' }]; - expect( - await vendir.updateArtifacts({ - packageFileName: 'vendir.yml', - updatedDeps, - newPackageFileContent: vendirFile, - config: { - ...config, - }, - }), - ).toEqual([ { file: { type: 'addition', diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 94cf79018945cc..5f3b47bed89b51 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -5,7 +5,6 @@ import type { ExecOptions } from '../../../util/exec/types'; import { getParentDir, getSiblingFileName, - localPathIsFile, readLocalFile, writeLocalFile, } from '../../../util/fs'; @@ -59,6 +58,7 @@ export async function updateArtifacts({ }); } else { logger.debug('vendir.lock.yml is unchanged'); + return null; } // add modified vendir archives to artifacts @@ -66,26 +66,34 @@ export async function updateArtifacts({ // Files must be in the vendor path to get added const vendorDir = getParentDir(packageFileName); const status = await getRepoStatus(); - for (const f of (status.modified ?? []).concat(status.not_added)) { - const isFileInVendorDir = f.startsWith(vendorDir); - if (vendorDir || isFileInVendorDir) { + if (status) { + const modifiedFiles = status.modified ?? []; + const notAddedFiles = status.not_added ?? []; + const deletedFiles = status.deleted ?? []; + + for (const f of modifiedFiles.concat(notAddedFiles)) { + const isFileInVendorDir = f.startsWith(vendorDir); + if (vendorDir || isFileInVendorDir) { + fileChanges.push({ + file: { + type: 'addition', + path: f, + contents: await readLocalFile(f), + }, + }); + } + } + + for (const f of deletedFiles) { fileChanges.push({ file: { - type: 'addition', + type: 'deletion', path: f, - contents: await readLocalFile(f), }, }); } - } - - for (const f of status.deleted ?? []) { - fileChanges.push({ - file: { - type: 'deletion', - path: f, - }, - }); + } else { + logger.error('Failed to get git status'); } return fileChanges.length > 0 ? fileChanges : null; diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index d4402cb194f6a1..3ddf423aaca118 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -1,5 +1,4 @@ - -Renovate supports updating Helm Chart references in vendir.yml via the [vendir](https://carvel.dev/vendir/) tool +Renovate supports updating Helm Chart references in vendir.yml via the [vendir](https://carvel.dev/vendir/) tool. Renovate requires the presence of a [vendir lock file](https://carvel.dev/vendir/docs/v0.40.x/vendir-lock-spec/) which is generated by vendir and should be stored in source code. It supports both https and oci helm chart repositories. @@ -28,14 +27,3 @@ helmChart: # '3' means binary 'helm3' needs to be on the path (optional) helmVersion: '3' ``` - -### Subchart Archives - -To get updates for subchart archives put `vendirUpdateSubChartArchives` in your `postUpdateOptions` configuration. -Renovate now updates archives in the folder specified by your vendir.yml. - -```json -{ - "postUpdateOptions": ["vendirUpdateSubChartArchives"] -} -``` From b89220e359e0fd2732225b9028bef7a31928efa4 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Fri, 15 Mar 2024 21:35:17 +0100 Subject: [PATCH 21/52] feat: attempt to support docker and install mode --- lib/modules/manager/vendir/artifacts.spec.ts | 133 ++++++++++++++++++- lib/modules/manager/vendir/artifacts.ts | 1 + 2 files changed, 133 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index cc985d261ffede..14e09cd65fe18f 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -2,13 +2,16 @@ import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../../test/exec-util'; import { Fixtures } from '../../../../test/fixtures'; -import { env, fs, git, partial } from '../../../../test/util'; +import { env, fs, git, mocked, partial } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; import type { StatusResult } from '../../../util/git/types'; +import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; import * as vendir from '.'; +const datasource = mocked(_datasource); + jest.mock('../../datasource', () => mockDeep()); jest.mock('../../../util/exec/env', () => mockDeep()); jest.mock('../../../util/http', () => mockDeep()); @@ -295,4 +298,132 @@ describe('modules/manager/vendir/artifacts', () => { }, ]); }); + + it('works explicit global binarySource', async () => { + GlobalConfig.set({ ...adminConfig, binarySource: 'global' }); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + const execSnapshots = mockExecAll(); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache', + ); + fs.getParentDir.mockReturnValue(''); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }), + ).toEqual([ + { + file: { + type: 'addition', + path: 'vendir.lock.yml', + contents: vendirLockFile2, + }, + }, + ]); + expect(execSnapshots).toMatchObject([{ cmd: 'vendir sync' }]); + }); + + it('supports install mode', async () => { + GlobalConfig.set({ + ...adminConfig, + binarySource: 'install', + }); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); + fs.readLocalFile.mockResolvedValueOnce('0.35.0'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache', + ); + fs.getParentDir.mockReturnValue(''); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }), + ).toEqual([ + { + file: { + type: 'addition', + path: 'vendir.lock.yml', + contents: vendirLockFile2, + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { cmd: 'install-tool vendir 0.35.0' }, + { cmd: 'vendir sync' }, + ]); + }); + + describe('Docker', () => { + beforeEach(() => { + GlobalConfig.set({ + ...adminConfig, + binarySource: 'docker', + }); + }); + + it('returns updated vendir.yml for lockfile maintenance', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); + fs.readLocalFile.mockResolvedValueOnce('0.35.0'); + const execSnapshots = mockExecAll(); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache', + ); + fs.getParentDir.mockReturnValue(''); + const updatedDeps = [{ depName: 'dep1' }]; + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }), + ).toEqual([ + { + file: { + type: 'addition', + path: 'vendir.lock.yml', + contents: vendirLockFile2, + }, + }, + ]); + expect(execSnapshots).toMatchObject([ + { cmd: 'install-tool vendir 0.35.0' }, + { cmd: 'vendir sync' }, + ]); + expect(execSnapshots).toMatchObject([ + { cmd: 'docker pull ghcr.io/containerbase/sidecar' }, + { cmd: 'docker ps --filter name=renovate_sidecar -aq' }, + { + cmd: + 'docker run --rm --name=renovate_sidecar --label=renovate_child ' + + '-v "/tmp/github/some/repo":"/tmp/github/some/repo" ' + + '-v "/tmp/cache":"/tmp/cache" ' + + '-e GEM_HOME ' + + '-e CONTAINERBASE_CACHE_DIR ' + + '-w "/tmp/github/some/repo" ' + + 'ghcr.io/containerbase/sidecar' + + ' bash -l -c "' + + 'install-tool vendir 0.35.0' + + ' && ' + + 'vendir sync' + + '"', + }, + ]); + }); + }); }); diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 5f3b47bed89b51..a198eca790b79c 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -34,6 +34,7 @@ export async function updateArtifacts({ logger.debug('Updating Vendir artifacts'); const execOptions: ExecOptions = { cwdFile: packageFileName, + docker: {}, toolConstraints: [ { toolName: 'vendir', constraint: config.constraints?.vendir }, { toolName: 'helm', constraint: config.constraints?.helm }, From 8297e0eb2ab7187994ac2b13a228b6716d1f7b4e Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 08:43:01 +0100 Subject: [PATCH 22/52] tests: test for install mode and docker mode --- lib/modules/manager/vendir/artifacts.spec.ts | 76 ++++++++++++++++---- 1 file changed, 64 insertions(+), 12 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 14e09cd65fe18f..ede987b0e47c6b 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -11,6 +11,7 @@ import type { UpdateArtifactsConfig } from '../types'; import * as vendir from '.'; const datasource = mocked(_datasource); +process.env.CONTAINERBASE = 'true'; jest.mock('../../datasource', () => mockDeep()); jest.mock('../../../util/exec/env', () => mockDeep()); @@ -21,7 +22,8 @@ jest.mock('../../../util/git', () => mockDeep()); const adminConfig: RepoGlobalConfig = { localDir: join('/tmp/github/some/repo'), // `join` fixes Windows CI cacheDir: join('/tmp/renovate/cache'), - containerbaseDir: join('/tmp/renovate/cache/containerbase'), + containerbaseDir: join('/tmp/cache/containerbase'), + dockerSidecarImage: 'ghcr.io/containerbase/sidecar', }; const config: UpdateArtifactsConfig = {}; @@ -238,7 +240,7 @@ describe('modules/manager/vendir/artifacts', () => { ]); }); - it('add artifacts without old archives', async () => { + it('add artifacts', async () => { fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); @@ -342,6 +344,9 @@ describe('modules/manager/vendir/artifacts', () => { fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [{ version: '0.35.0' }], + }); fs.getParentDir.mockReturnValue(''); const updatedDeps = [{ depName: 'dep1' }]; expect( @@ -349,7 +354,10 @@ describe('modules/manager/vendir/artifacts', () => { packageFileName: 'vendir.yml', updatedDeps, newPackageFileContent: vendirFile, - config, + config: { + ...config, + constraints: { vendir: '0.35.0', helm: '3.17.0' }, + }, }), ).toEqual([ { @@ -361,8 +369,48 @@ describe('modules/manager/vendir/artifacts', () => { }, ]); expect(execSnapshots).toMatchObject([ - { cmd: 'install-tool vendir 0.35.0' }, - { cmd: 'vendir sync' }, + { + cmd: 'install-tool vendir 0.35.0', + options: { + env: { + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', + }, + }, + }, + { + cmd: 'install-tool helm 3.17.0', + options: { + env: { + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', + }, + }, + }, + { + cmd: 'vendir sync', + options: { + env: { + HOME: '/home/user', + HTTPS_PROXY: 'https://example.com', + HTTP_PROXY: 'http://example.com', + LANG: 'en_US.UTF-8', + LC_ALL: 'en_US', + NO_PROXY: 'localhost', + PATH: '/tmp/path', + }, + }, + }, ]); }); @@ -379,6 +427,9 @@ describe('modules/manager/vendir/artifacts', () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); fs.readLocalFile.mockResolvedValueOnce('0.35.0'); + datasource.getPkgReleases.mockResolvedValueOnce({ + releases: [{ version: '0.35.0' }], + }); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', @@ -390,7 +441,10 @@ describe('modules/manager/vendir/artifacts', () => { packageFileName: 'vendir.yml', updatedDeps, newPackageFileContent: vendirFile, - config, + config: { + ...config, + constraints: { vendir: '0.35.0', helm: '3.17.0' }, + }, }), ).toEqual([ { @@ -401,10 +455,6 @@ describe('modules/manager/vendir/artifacts', () => { }, }, ]); - expect(execSnapshots).toMatchObject([ - { cmd: 'install-tool vendir 0.35.0' }, - { cmd: 'vendir sync' }, - ]); expect(execSnapshots).toMatchObject([ { cmd: 'docker pull ghcr.io/containerbase/sidecar' }, { cmd: 'docker ps --filter name=renovate_sidecar -aq' }, @@ -412,14 +462,16 @@ describe('modules/manager/vendir/artifacts', () => { cmd: 'docker run --rm --name=renovate_sidecar --label=renovate_child ' + '-v "/tmp/github/some/repo":"/tmp/github/some/repo" ' + - '-v "/tmp/cache":"/tmp/cache" ' + - '-e GEM_HOME ' + + '-v "/tmp/renovate/cache":"/tmp/renovate/cache" ' + + '-v "/tmp/cache/containerbase":"/tmp/cache/containerbase" ' + '-e CONTAINERBASE_CACHE_DIR ' + '-w "/tmp/github/some/repo" ' + 'ghcr.io/containerbase/sidecar' + ' bash -l -c "' + 'install-tool vendir 0.35.0' + ' && ' + + 'install-tool helm 3.17.0' + + ' && ' + 'vendir sync' + '"', }, From 7e1ef172c04f5c9cd61d33ed388a85518fb56568 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 09:04:36 +0100 Subject: [PATCH 23/52] tests: add test for Temporary Error --- lib/modules/manager/vendir/artifacts.spec.ts | 28 ++++++++++++++++++++ lib/modules/manager/vendir/artifacts.ts | 1 - 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index ede987b0e47c6b..96a606e02fd074 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -5,6 +5,8 @@ import { Fixtures } from '../../../../test/fixtures'; import { env, fs, git, mocked, partial } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; +import { TEMPORARY_ERROR } from '../../../constants/error-messages'; +import { ExecError } from '../../../util/exec/exec-error'; import type { StatusResult } from '../../../util/git/types'; import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; @@ -166,6 +168,32 @@ describe('modules/manager/vendir/artifacts', () => { ]); }); + it('rethrows for temporary error', async () => { + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); + fs.getSiblingFileName.mockReturnValueOnce('vendir.yml'); + fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); + fs.privateCacheDir.mockReturnValue( + '/tmp/renovate/cache/__renovate-private-cache', + ); + fs.getParentDir.mockReturnValue(''); + const execError = new ExecError(TEMPORARY_ERROR, { + cmd: '', + stdout: '', + stderr: '', + options: { encoding: 'utf8' }, + }); + const updatedDeps = [{ depName: 'dep1' }]; + mockExecAll(execError); + await expect( + vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: vendirFile, + config, + }), + ).rejects.toThrow(TEMPORARY_ERROR); + }); + it('add artifacts to file list if vendir.yml exists', async () => { fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index a198eca790b79c..35b03c68306e84 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -99,7 +99,6 @@ export async function updateArtifacts({ return fileChanges.length > 0 ? fileChanges : null; } catch (err) { - // istanbul ignore if if (err.message === TEMPORARY_ERROR) { throw err; } From 10db10f6f410ff78c20b4f7b82d1724a972fb39d Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 09:19:43 +0100 Subject: [PATCH 24/52] Update lib/modules/manager/vendir/extract.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index dadb245eb7df06..ed72b2123f0ffe 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -47,7 +47,7 @@ export function parseVendir( export function extractPackageFile( content: string, - packageFile?: string, + packageFile: string, ): PackageFileContent | null { logger.trace(`vendir.extractPackageFile(${packageFile!})`); const deps: PackageDependency[] = []; From ad59afe3541ffb7cc4130ada843a9f7f328bac0f Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 09:35:50 +0100 Subject: [PATCH 25/52] tests: clean up unecessary declarations --- lib/modules/manager/vendir/artifacts.spec.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 96a606e02fd074..774c9d0e2a5aca 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -2,17 +2,15 @@ import { mockDeep } from 'jest-mock-extended'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../../test/exec-util'; import { Fixtures } from '../../../../test/fixtures'; -import { env, fs, git, mocked, partial } from '../../../../test/util'; +import { env, fs, git, partial } from '../../../../test/util'; import { GlobalConfig } from '../../../config/global'; import type { RepoGlobalConfig } from '../../../config/types'; import { TEMPORARY_ERROR } from '../../../constants/error-messages'; import { ExecError } from '../../../util/exec/exec-error'; import type { StatusResult } from '../../../util/git/types'; -import * as _datasource from '../../datasource'; import type { UpdateArtifactsConfig } from '../types'; import * as vendir from '.'; -const datasource = mocked(_datasource); process.env.CONTAINERBASE = 'true'; jest.mock('../../datasource', () => mockDeep()); @@ -367,14 +365,10 @@ describe('modules/manager/vendir/artifacts', () => { fs.readLocalFile.mockResolvedValueOnce(vendirLockFile1); fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); - fs.readLocalFile.mockResolvedValueOnce('0.35.0'); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', ); - datasource.getPkgReleases.mockResolvedValueOnce({ - releases: [{ version: '0.35.0' }], - }); fs.getParentDir.mockReturnValue(''); const updatedDeps = [{ depName: 'dep1' }]; expect( @@ -455,9 +449,6 @@ describe('modules/manager/vendir/artifacts', () => { fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); fs.readLocalFile.mockResolvedValueOnce(vendirLockFile2); fs.readLocalFile.mockResolvedValueOnce('0.35.0'); - datasource.getPkgReleases.mockResolvedValueOnce({ - releases: [{ version: '0.35.0' }], - }); const execSnapshots = mockExecAll(); fs.privateCacheDir.mockReturnValue( '/tmp/renovate/cache/__renovate-private-cache', From 5090b16821c4173e9028fdb9150405f93927b00f Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 09:36:03 +0100 Subject: [PATCH 26/52] fix: add DockerDatasource to index.ts --- lib/modules/manager/vendir/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/index.ts b/lib/modules/manager/vendir/index.ts index de9d67d3fc3400..aaab07563e9f87 100644 --- a/lib/modules/manager/vendir/index.ts +++ b/lib/modules/manager/vendir/index.ts @@ -1,3 +1,4 @@ +import { DockerDatasource } from '../../datasource/docker'; import { HelmDatasource } from '../../datasource/helm'; export { extractPackageFile } from './extract'; export { updateArtifacts } from './artifacts'; @@ -7,5 +8,5 @@ export const defaultConfig = { fileMatch: ['(^|/)vendir\\.yml$'], }; -export const supportedDatasources = [HelmDatasource.id]; +export const supportedDatasources = [HelmDatasource.id, DockerDatasource.id]; export const supportsLockFileMaintenance = true; From d016a703000b6c4dca99c26bd8e4c13235868d69 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 09:39:11 +0100 Subject: [PATCH 27/52] docs: add complete vendir.yml --- lib/modules/manager/vendir/readme.md | 86 ++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 3ddf423aaca118..5db8bbd1ca6c4f 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -2,28 +2,66 @@ Renovate supports updating Helm Chart references in vendir.yml via the [vendir]( It supports both https and oci helm chart repositories. -```yaml title="Example helmChart" -# fetch Helm chart contents (optional; v0.11.0+) -helmChart: - # chart name (required) - name: stable/redis - # use specific chart version (string; optional) - version: '1.2.1' - # specifies Helm repository to fetch from (optional) - repository: - # repository url; supports exprimental oci helm fetch via - # oci:// scheme (required) - url: https://... - # specifies name of a secret with helm repo auth details; - # secret may include 'username', 'password'; - # as of v0.19.0+, dockerconfigjson secrets are also supported (optional) - # as of v0.22.0+, 0 or 1 auth credential is expected within dockerconfigjson secret - # if >1 auth creds found, error will be returned. (currently registry hostname - # is not used when found in provide auth credential.) - secretRef: - # (required) - name: my-helm-auth - # specify helm binary version to use; - # '3' means binary 'helm3' needs to be on the path (optional) - helmVersion: '3' +```yaml title="Example vendir.yml" +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config + +# declaration of minimum required vendir binary version (optional) +minimumRequiredVersion: 0.8.0 + +# one or more directories to manage with vendir +directories: +- # path is relative to `vendir` CLI working directory + path: config/_ytt_lib + + # set the permissions for this directory (optional; v0.33.0+) + # by default directories will be created with 0700 + # can be provided as octal, in which case it needs to be prefixed with a `0` + permissions: 0700 + + contents: + - # path lives relative to directory path # (required) + path: github.com/cloudfoundry/cf-k8s-networking + + # skip fetching if the config for this path has not changed since the last sync + # optional, `false` by default, available since v0.36.0 + # use `vendir sync --lazy=false` to forcefully sync when needed + lazy: true + + # fetch Helm chart contents (optional; v0.11.0+) + helmChart: + # chart name (required) + name: stable/redis + # use specific chart version (string; optional) + version: "1.2.1" + # specifies Helm repository to fetch from (optional) + repository: + # repository url; supports exprimental oci helm fetch via + # oci:// scheme (required) + url: https://... + # specify helm binary version to use; + # '3' means binary 'helm3' needs to be on the path (optional) + helmVersion: "3" + + # includes paths specify what should be included. by default + # all paths are included (optional) + includePaths: + - cfroutesync/crds/**/* + - install/ytt/networking/**/* + + # exclude paths are "placed" on top of include paths (optional) + excludePaths: [] + + # specifies paths to files that need to be includes for + # legal reasons such as LICENSE file. Defaults to few + # LICENSE, NOTICE and COPYRIGHT variations (optional) + legalPaths: [] + + # make subdirectory to be new root path within this asset (optional; v0.11.0+) + newRootPath: cfroutesync + + # set the permissions for this content directory (optional; v0.33.0+) + # by default content directories will be created with 0700 + # can be provided as octal, in which case it needs to be prefixed with a `0` + permissions: 0700 ``` From a5fd962e8e69739f8391740942dba4f8b46b2554 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 14:40:21 +0100 Subject: [PATCH 28/52] feat: add support for registryAliases feat: swap out non-vendir specific functions for well tested helm functions test: add test for registryAliases --- .../vendir/__fixtures__/alias-contents.yaml | 30 ++++++++++ .../vendir/__fixtures__/non-helmchart.yaml | 12 ++++ lib/modules/manager/vendir/artifacts.spec.ts | 15 +++++ lib/modules/manager/vendir/artifacts.ts | 3 +- lib/modules/manager/vendir/extract.spec.ts | 59 ++++++++++++++++--- lib/modules/manager/vendir/extract.ts | 49 +++++++++++---- lib/modules/manager/vendir/utils.ts | 4 -- 7 files changed, 148 insertions(+), 24 deletions(-) create mode 100644 lib/modules/manager/vendir/__fixtures__/alias-contents.yaml create mode 100644 lib/modules/manager/vendir/__fixtures__/non-helmchart.yaml diff --git a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml new file mode 100644 index 00000000000000..9b0ae047d58b5b --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: custom-repo-custom-version + helmChart: + name: oci-contour + version: "7.10.1" + repository: + url: alias:ociRegistry + - path: custom-repo-custom-version + helmChart: + name: normal-contour + version: "7.10.1" + repository: + url: alias:longAlias + - path: custom-repo-custom-version + helmChart: + name: placeholder-contour + version: "7.10.1" + repository: + url: '@placeholder' + - path: custom-repo-custom-version + helmChart: + name: broken-contour + version: "7.10.1" + repository: + url: alias:brokenAlias diff --git a/lib/modules/manager/vendir/__fixtures__/non-helmchart.yaml b/lib/modules/manager/vendir/__fixtures__/non-helmchart.yaml new file mode 100644 index 00000000000000..738c9257cad17f --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/non-helmchart.yaml @@ -0,0 +1,12 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: +- path: vendor + contents: + - path: github.com/cloudfoundry/cf-k8s-networking + git: + # http or ssh urls are supported (required) + url: https://github.com/cloudfoundry/cf-k8s-networking + # branch, tag, commit; origin is the name of the remote (required) + # optional if refSelection is specified (available in v0.11.0+) + ref: origin/master diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 774c9d0e2a5aca..57f102f410e575 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -30,6 +30,7 @@ const config: UpdateArtifactsConfig = {}; const vendirLockFile1 = Fixtures.get('vendir_1.lock'); const vendirLockFile2 = Fixtures.get('vendir_2.lock'); const vendirFile = Fixtures.get('vendir.yml'); +const emptyVendirLock = ''; describe('modules/manager/vendir/artifacts', () => { beforeEach(() => { @@ -49,6 +50,20 @@ describe('modules/manager/vendir/artifacts', () => { ).toBeNull(); }); + it('returns null if empty vendir.lock.yml found', async () => { + const updatedDeps = [{ depName: 'dep1' }]; + fs.readLocalFile.mockResolvedValueOnce(''); + fs.getSiblingFileName.mockReturnValueOnce('vendir.lock.yml'); + expect( + await vendir.updateArtifacts({ + packageFileName: 'vendir.yml', + updatedDeps, + newPackageFileContent: '', + config, + }), + ).toBeNull(); + }); + it('returns null if updatedDeps is empty', async () => { expect( await vendir.updateArtifacts({ diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 35b03c68306e84..0b5cccfc61b8ea 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -21,7 +21,7 @@ export async function updateArtifacts({ const lockFileName = getSiblingFileName(packageFileName, 'vendir.lock.yml'); if (!lockFileName) { - logger.debug('No vendir.lock.yml found'); + logger.warn('No vendir.lock.yml found'); return null; } const existingLockFileContent = await readLocalFile(lockFileName, 'utf8'); @@ -29,6 +29,7 @@ export async function updateArtifacts({ logger.debug('No vendir.lock.yml found'); return null; } + try { await writeLocalFile(packageFileName, newPackageFileContent); logger.debug('Updating Vendir artifacts'); diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index e41a97b767e582..2d832b3002aa02 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -3,7 +3,9 @@ import { extractPackageFile } from '.'; const oneContents = Fixtures.get('one-contents.yaml'); const ociContents = Fixtures.get('oci-contents.yaml'); +const aliasContents = Fixtures.get('alias-contents.yaml'); const multipleContents = Fixtures.get('multiple-contents.yaml'); +const nonHelmChartContents = Fixtures.get('non-helmchart.yaml'); const emptyDirectories = Fixtures.mock( { apiVersion: 'vendir.k14s.io/v1alpha1', kind: 'Config', directories: [] }, 'vendir.yml', @@ -12,22 +14,27 @@ const emptyDirectories = Fixtures.mock( describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { it('returns null for invalid yaml file content', () => { - const result = extractPackageFile('nothing here: ['); + const result = extractPackageFile('nothing here: [', 'vendir.yml', {}); expect(result).toBeNull(); }); it('returns null for empty yaml file content', () => { - const result = extractPackageFile(''); + const result = extractPackageFile('', 'vendir.yml', {}); expect(result).toBeNull(); }); it('returns null for empty directories key', () => { - const result = extractPackageFile(emptyDirectories); + const result = extractPackageFile(emptyDirectories, 'vendir.yml', {}); + expect(result).toBeNull(); + }); + + it('returns null for nonHelmChart key', () => { + const result = extractPackageFile(nonHelmChartContents, 'vendir.yml', {}); expect(result).toBeNull(); }); it('single chart - extracts helm-chart from vendir.yml correctly', () => { - const result = extractPackageFile(oneContents); + const result = extractPackageFile(oneContents, 'vendir.yml', {}); expect(result).toMatchObject({ deps: [ { @@ -41,21 +48,20 @@ describe('modules/manager/vendir/extract', () => { }); it('single chart - extracts oci helm-chart from vendir.yml correctly', () => { - const result = extractPackageFile(ociContents); + const result = extractPackageFile(ociContents, 'vendir.yml', {}); expect(result).toMatchObject({ deps: [ { currentValue: '7.10.1', depName: 'contour', datasource: 'docker', - registryUrls: ['https://charts.bitnami.com/bitnami'], }, ], }); }); it('multiple charts - extracts helm-chart from vendir.yml correctly', () => { - const result = extractPackageFile(multipleContents); + const result = extractPackageFile(multipleContents, 'vendir.yml', {}); expect(result).toMatchObject({ deps: [ { @@ -73,5 +79,44 @@ describe('modules/manager/vendir/extract', () => { ], }); }); + + it('resolves aliased registry urls', async () => { + const ociresult = extractPackageFile(aliasContents, 'vendir.yml', { + registryAliases: { + placeholder: 'https://my-registry.gcr.io/', + longAlias: 'https://registry.example.com/', + ociRegistry: 'oci://quay.example.com/organization', + }, + }); + + expect(ociresult).toMatchObject({ + deps: [ + { + currentValue: '7.10.1', + depName: 'oci-contour', + datasource: 'helm', + registryUrls: ['oci://quay.example.com/organization'], + }, + { + currentValue: '7.10.1', + depName: 'normal-contour', + datasource: 'helm', + registryUrls: ['https://registry.example.com/'], + }, + { + currentValue: '7.10.1', + depName: 'placeholder-contour', + datasource: 'helm', + registryUrls: ['https://my-registry.gcr.io/'], + }, + { + currentValue: '7.10.1', + depName: 'broken-contour', + datasource: 'helm', + skipReason: 'placeholder-url', + }, + ], + }); + }); }); }); diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index ed72b2123f0ffe..3de483c4018eef 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -1,29 +1,53 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; import { parseSingleYaml } from '../../../util/yaml'; -import { DockerDatasource } from '../../datasource/docker'; import { HelmDatasource } from '../../datasource/helm'; -import type { PackageDependency, PackageFileContent } from '../types'; +import { getDep } from '../dockerfile/extract'; +import { isOCIRegistry, resolveAlias } from '../helmv3/utils'; +import type { + ExtractConfig, + PackageDependency, + PackageFileContent, +} from '../types'; import type { HelmChart, Vendir } from './types'; -import { isHelmChart, isOCIRegistry } from './utils'; +import { isHelmChart } from './utils'; // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. export function extractHelmChart( helmChart: HelmChart, + aliases?: Record | undefined, ): PackageDependency | null { - let registryUrl = helmChart.repository.url; - let dataSource = HelmDatasource.id; - if (isOCIRegistry(helmChart.repository)) { - registryUrl = helmChart.repository.url.replace('oci://', 'https://'); - dataSource = DockerDatasource.id; + if (isOCIRegistry(helmChart.repository.url)) { + const dep = getDep( + `${helmChart.repository.url.replace('oci://', '')}/${helmChart.name}:${helmChart.version}`, + false, + aliases, + ); + return { + ...dep, + depName: helmChart.name, + packageName: dep.depName, + // https://github.com/helm/helm/issues/10312 + // https://github.com/helm/helm/issues/10678 + pinDigests: false, + }; + } + const repository = resolveAlias(helmChart.repository.url, aliases!); + if (!repository) { + return { + depName: helmChart.name, + currentValue: helmChart.version, + datasource: HelmDatasource.id, + skipReason: 'placeholder-url', + }; } return { depName: helmChart.name, currentValue: helmChart.version, - registryUrls: [registryUrl], - datasource: dataSource, + registryUrls: [repository], + datasource: HelmDatasource.id, }; } @@ -48,8 +72,9 @@ export function parseVendir( export function extractPackageFile( content: string, packageFile: string, + config: ExtractConfig, ): PackageFileContent | null { - logger.trace(`vendir.extractPackageFile(${packageFile!})`); + logger.trace(`vendir.extractPackageFile(${packageFile})`); const deps: PackageDependency[] = []; const pkg = parseVendir(content, packageFile); @@ -61,7 +86,7 @@ export function extractPackageFile( const contents = pkg.directories.flatMap((directory) => directory.contents); for (const content of contents) { if (isHelmChart(content)) { - const dep = extractHelmChart(content.helmChart); + const dep = extractHelmChart(content.helmChart, config.registryAliases); if (dep) { deps.push({ ...dep, diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index c9ec6eb6f2347a..07d3779bfa1635 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -3,7 +3,3 @@ import type { Contents, HelmChartContent, Repository } from './types'; export function isHelmChart(item: Contents): item is HelmChartContent { return 'helmChart' in item; } - -export function isOCIRegistry(repository: Repository): boolean { - return repository.url.startsWith('oci://'); -} From b115e9440024503fc3a2313c4db48e3a910d556d Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 14:56:40 +0100 Subject: [PATCH 29/52] fix: increase log level for misconfigured lockfiles fix: change wording for warn message --- lib/modules/manager/vendir/artifacts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 0b5cccfc61b8ea..f91bb311f472f9 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -26,7 +26,7 @@ export async function updateArtifacts({ } const existingLockFileContent = await readLocalFile(lockFileName, 'utf8'); if (!existingLockFileContent) { - logger.debug('No vendir.lock.yml found'); + logger.warn('Empty vendir.lock.yml found'); return null; } From c92bd8fdc977d1279594a4df7939b82a35c182f5 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 15:10:59 +0100 Subject: [PATCH 30/52] fix: lint errors --- lib/modules/manager/vendir/artifacts.spec.ts | 1 - lib/modules/manager/vendir/extract.spec.ts | 2 +- lib/modules/manager/vendir/readme.md | 88 ++++++++++---------- lib/modules/manager/vendir/utils.ts | 2 +- 4 files changed, 46 insertions(+), 47 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.spec.ts b/lib/modules/manager/vendir/artifacts.spec.ts index 57f102f410e575..32d7ce8efdc23b 100644 --- a/lib/modules/manager/vendir/artifacts.spec.ts +++ b/lib/modules/manager/vendir/artifacts.spec.ts @@ -30,7 +30,6 @@ const config: UpdateArtifactsConfig = {}; const vendirLockFile1 = Fixtures.get('vendir_1.lock'); const vendirLockFile2 = Fixtures.get('vendir_2.lock'); const vendirFile = Fixtures.get('vendir.yml'); -const emptyVendirLock = ''; describe('modules/manager/vendir/artifacts', () => { beforeEach(() => { diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 2d832b3002aa02..584c46e1217915 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -80,7 +80,7 @@ describe('modules/manager/vendir/extract', () => { }); }); - it('resolves aliased registry urls', async () => { + it('resolves aliased registry urls', () => { const ociresult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { placeholder: 'https://my-registry.gcr.io/', diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 5db8bbd1ca6c4f..eac18cd0bcbeeb 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -11,57 +11,57 @@ minimumRequiredVersion: 0.8.0 # one or more directories to manage with vendir directories: -- # path is relative to `vendir` CLI working directory - path: config/_ytt_lib + - # path is relative to `vendir` CLI working directory + path: config/_ytt_lib - # set the permissions for this directory (optional; v0.33.0+) - # by default directories will be created with 0700 - # can be provided as octal, in which case it needs to be prefixed with a `0` - permissions: 0700 + # set the permissions for this directory (optional; v0.33.0+) + # by default directories will be created with 0700 + # can be provided as octal, in which case it needs to be prefixed with a `0` + permissions: 0700 - contents: - - # path lives relative to directory path # (required) - path: github.com/cloudfoundry/cf-k8s-networking + contents: + - # path lives relative to directory path # (required) + path: github.com/cloudfoundry/cf-k8s-networking - # skip fetching if the config for this path has not changed since the last sync - # optional, `false` by default, available since v0.36.0 - # use `vendir sync --lazy=false` to forcefully sync when needed - lazy: true + # skip fetching if the config for this path has not changed since the last sync + # optional, `false` by default, available since v0.36.0 + # use `vendir sync --lazy=false` to forcefully sync when needed + lazy: true - # fetch Helm chart contents (optional; v0.11.0+) - helmChart: - # chart name (required) - name: stable/redis - # use specific chart version (string; optional) - version: "1.2.1" - # specifies Helm repository to fetch from (optional) - repository: - # repository url; supports exprimental oci helm fetch via - # oci:// scheme (required) - url: https://... - # specify helm binary version to use; - # '3' means binary 'helm3' needs to be on the path (optional) - helmVersion: "3" + # fetch Helm chart contents (optional; v0.11.0+) + helmChart: + # chart name (required) + name: stable/redis + # use specific chart version (string; optional) + version: '1.2.1' + # specifies Helm repository to fetch from (optional) + repository: + # repository url; supports exprimental oci helm fetch via + # oci:// scheme (required) + url: https://... + # specify helm binary version to use; + # '3' means binary 'helm3' needs to be on the path (optional) + helmVersion: '3' - # includes paths specify what should be included. by default - # all paths are included (optional) - includePaths: - - cfroutesync/crds/**/* - - install/ytt/networking/**/* + # includes paths specify what should be included. by default + # all paths are included (optional) + includePaths: + - cfroutesync/crds/**/* + - install/ytt/networking/**/* - # exclude paths are "placed" on top of include paths (optional) - excludePaths: [] + # exclude paths are "placed" on top of include paths (optional) + excludePaths: [] - # specifies paths to files that need to be includes for - # legal reasons such as LICENSE file. Defaults to few - # LICENSE, NOTICE and COPYRIGHT variations (optional) - legalPaths: [] + # specifies paths to files that need to be includes for + # legal reasons such as LICENSE file. Defaults to few + # LICENSE, NOTICE and COPYRIGHT variations (optional) + legalPaths: [] - # make subdirectory to be new root path within this asset (optional; v0.11.0+) - newRootPath: cfroutesync + # make subdirectory to be new root path within this asset (optional; v0.11.0+) + newRootPath: cfroutesync - # set the permissions for this content directory (optional; v0.33.0+) - # by default content directories will be created with 0700 - # can be provided as octal, in which case it needs to be prefixed with a `0` - permissions: 0700 + # set the permissions for this content directory (optional; v0.33.0+) + # by default content directories will be created with 0700 + # can be provided as octal, in which case it needs to be prefixed with a `0` + permissions: 0700 ``` diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts index 07d3779bfa1635..d721141571d18f 100644 --- a/lib/modules/manager/vendir/utils.ts +++ b/lib/modules/manager/vendir/utils.ts @@ -1,4 +1,4 @@ -import type { Contents, HelmChartContent, Repository } from './types'; +import type { Contents, HelmChartContent } from './types'; export function isHelmChart(item: Contents): item is HelmChartContent { return 'helmChart' in item; From 4f608a9e12a1b5124bd2270b8a6269bb3987a056 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 15:20:01 +0100 Subject: [PATCH 31/52] fix: type errors --- .../manager/vendir/__fixtures__/empty-directories.yaml | 3 +++ lib/modules/manager/vendir/extract.spec.ts | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 lib/modules/manager/vendir/__fixtures__/empty-directories.yaml diff --git a/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml b/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml new file mode 100644 index 00000000000000..b9e824668b4657 --- /dev/null +++ b/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml @@ -0,0 +1,3 @@ +apiVersion: vendir.k14s.io/v1alpha1 +kind: Config +directories: [] diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 584c46e1217915..fba13254e76f88 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -6,10 +6,7 @@ const ociContents = Fixtures.get('oci-contents.yaml'); const aliasContents = Fixtures.get('alias-contents.yaml'); const multipleContents = Fixtures.get('multiple-contents.yaml'); const nonHelmChartContents = Fixtures.get('non-helmchart.yaml'); -const emptyDirectories = Fixtures.mock( - { apiVersion: 'vendir.k14s.io/v1alpha1', kind: 'Config', directories: [] }, - 'vendir.yml', -); +const emptyDirectories = Fixtures.get('empty-directories.yaml'); describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { From e014db3628d07c89601d392c3ffea9d782db4b77 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 15:45:47 +0100 Subject: [PATCH 32/52] docs: update example --- lib/modules/manager/vendir/readme.md | 39 ---------------------------- 1 file changed, 39 deletions(-) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index eac18cd0bcbeeb..cb7f84f16a3942 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -6,29 +6,12 @@ It supports both https and oci helm chart repositories. apiVersion: vendir.k14s.io/v1alpha1 kind: Config -# declaration of minimum required vendir binary version (optional) -minimumRequiredVersion: 0.8.0 - # one or more directories to manage with vendir directories: - # path is relative to `vendir` CLI working directory path: config/_ytt_lib - - # set the permissions for this directory (optional; v0.33.0+) - # by default directories will be created with 0700 - # can be provided as octal, in which case it needs to be prefixed with a `0` - permissions: 0700 - contents: - - # path lives relative to directory path # (required) path: github.com/cloudfoundry/cf-k8s-networking - - # skip fetching if the config for this path has not changed since the last sync - # optional, `false` by default, available since v0.36.0 - # use `vendir sync --lazy=false` to forcefully sync when needed - lazy: true - - # fetch Helm chart contents (optional; v0.11.0+) helmChart: # chart name (required) name: stable/redis @@ -42,26 +25,4 @@ directories: # specify helm binary version to use; # '3' means binary 'helm3' needs to be on the path (optional) helmVersion: '3' - - # includes paths specify what should be included. by default - # all paths are included (optional) - includePaths: - - cfroutesync/crds/**/* - - install/ytt/networking/**/* - - # exclude paths are "placed" on top of include paths (optional) - excludePaths: [] - - # specifies paths to files that need to be includes for - # legal reasons such as LICENSE file. Defaults to few - # LICENSE, NOTICE and COPYRIGHT variations (optional) - legalPaths: [] - - # make subdirectory to be new root path within this asset (optional; v0.11.0+) - newRootPath: cfroutesync - - # set the permissions for this content directory (optional; v0.33.0+) - # by default content directories will be created with 0700 - # can be provided as octal, in which case it needs to be prefixed with a `0` - permissions: 0700 ``` From 79f133ddf5363857252b8243ac137210a3f7a66b Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Sat, 16 Mar 2024 20:55:29 +0100 Subject: [PATCH 33/52] feat: add zod schema fix: remove ! from aliases --- lib/config/options/index.ts | 1 + lib/modules/manager/vendir/extract.ts | 19 +++++++++++------ lib/modules/manager/vendir/schema.ts | 30 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 lib/modules/manager/vendir/schema.ts diff --git a/lib/config/options/index.ts b/lib/config/options/index.ts index 6ab0b82551b743..a77f5df5837e9e 100644 --- a/lib/config/options/index.ts +++ b/lib/config/options/index.ts @@ -1034,6 +1034,7 @@ const options: RenovateOptions[] = [ 'kubernetes', 'kustomize', 'terraform', + 'vendir', 'woodpecker', ], }, diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 3de483c4018eef..823c11bb570768 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -9,14 +9,15 @@ import type { PackageDependency, PackageFileContent, } from '../types'; -import type { HelmChart, Vendir } from './types'; +import { HelmChartDefinition, Vendir, VendirDefinition } from './schema'; + import { isHelmChart } from './utils'; // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. export function extractHelmChart( - helmChart: HelmChart, + helmChart: HelmChartDefinition, aliases?: Record | undefined, ): PackageDependency | null { if (isOCIRegistry(helmChart.repository.url)) { @@ -34,7 +35,10 @@ export function extractHelmChart( pinDigests: false, }; } - const repository = resolveAlias(helmChart.repository.url, aliases!); + let repository = helmChart.repository.url || null; + if (aliases) { + repository = resolveAlias(helmChart.repository.url, aliases); + } if (!repository) { return { depName: helmChart.name, @@ -54,10 +58,13 @@ export function extractHelmChart( export function parseVendir( content: string, packageFile?: string, -): Vendir | null { - let pkg: Vendir | null = null; +): VendirDefinition | null { + let pkg: VendirDefinition[]; try { - pkg = parseSingleYaml(content); + pkg = parseSingleYaml(content, { + customSchema: Vendir, + removeTemplates: true, + }); } catch (e) /* istanbul ignore next */ { logger.debug({ packageFile }, 'Error parsing vendir.yml file'); return null; diff --git a/lib/modules/manager/vendir/schema.ts b/lib/modules/manager/vendir/schema.ts new file mode 100644 index 00000000000000..e4bf27567fbcf2 --- /dev/null +++ b/lib/modules/manager/vendir/schema.ts @@ -0,0 +1,30 @@ +import { z } from 'zod'; +import { LooseArray } from '../../../util/schema-utils'; + +export const KubernetesResource = z.object({ + apiVersion: z.literal('vendir.k14s.io/v1alpha1'), +}); + +export const HelmChart = z.object({ + name: z.string(), + version: z.string(), + repository: z.object({ url: z.string() }), +}); + +export const Contents = z.object({ + path: z.string(), + helmChart: LooseArray(HelmChart), +}); + +export const Vendir = KubernetesResource.extend({ + kind: z.literal('Config'), + directories: LooseArray( + z.object({ + path: z.string(), + contents: LooseArray(Contents), + }), + ), +}); + +export type VendirDefinition = z.infer; +export type HelmChartDefinition = z.infer; From 7b6c2692b38c7c0093da2fcce1f0d43641cfdc46 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 13:46:07 +0100 Subject: [PATCH 34/52] fix: improve schema --- lib/modules/manager/vendir/schema.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/modules/manager/vendir/schema.ts b/lib/modules/manager/vendir/schema.ts index e4bf27567fbcf2..a58aed066a354b 100644 --- a/lib/modules/manager/vendir/schema.ts +++ b/lib/modules/manager/vendir/schema.ts @@ -1,8 +1,9 @@ import { z } from 'zod'; import { LooseArray } from '../../../util/schema-utils'; -export const KubernetesResource = z.object({ +export const VendirResource = z.object({ apiVersion: z.literal('vendir.k14s.io/v1alpha1'), + kind: z.literal('Config'), }); export const HelmChart = z.object({ @@ -16,8 +17,7 @@ export const Contents = z.object({ helmChart: LooseArray(HelmChart), }); -export const Vendir = KubernetesResource.extend({ - kind: z.literal('Config'), +export const Vendir = VendirResource.extend({ directories: LooseArray( z.object({ path: z.string(), From e3a7a7043c5baa14e93c225d671c7711f6f03ac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Wed, 20 Mar 2024 14:37:11 +0100 Subject: [PATCH 35/52] fix typing issues --- lib/modules/manager/vendir/extract.ts | 2 +- lib/modules/manager/vendir/schema.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 823c11bb570768..065ab4d1491e1b 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -59,7 +59,7 @@ export function parseVendir( content: string, packageFile?: string, ): VendirDefinition | null { - let pkg: VendirDefinition[]; + let pkg: VendirDefinition; try { pkg = parseSingleYaml(content, { customSchema: Vendir, diff --git a/lib/modules/manager/vendir/schema.ts b/lib/modules/manager/vendir/schema.ts index a58aed066a354b..01472e1403ba35 100644 --- a/lib/modules/manager/vendir/schema.ts +++ b/lib/modules/manager/vendir/schema.ts @@ -9,16 +9,18 @@ export const VendirResource = z.object({ export const HelmChart = z.object({ name: z.string(), version: z.string(), - repository: z.object({ url: z.string() }), + repository: z.object({ + url: z.string(), + }), }); export const Contents = z.object({ path: z.string(), - helmChart: LooseArray(HelmChart), + helmChart: HelmChart, }); export const Vendir = VendirResource.extend({ - directories: LooseArray( + directories: z.array( z.object({ path: z.string(), contents: LooseArray(Contents), From eea8ff56e74e16149bb02fce12128388cb746d9f Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 14:44:33 +0100 Subject: [PATCH 36/52] fix: lint fix: coverage --- lib/modules/manager/vendir/extract.ts | 3 --- lib/modules/manager/vendir/readme.md | 28 +++++++++++++-------------- 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 065ab4d1491e1b..0d3df92fd547b7 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -70,9 +70,6 @@ export function parseVendir( return null; } - if (!pkg || is.string(pkg)) { - return null; - } return pkg; } diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index cb7f84f16a3942..de5b19c17ad2e1 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -11,18 +11,18 @@ directories: - # path is relative to `vendir` CLI working directory path: config/_ytt_lib contents: - path: github.com/cloudfoundry/cf-k8s-networking - helmChart: - # chart name (required) - name: stable/redis - # use specific chart version (string; optional) - version: '1.2.1' - # specifies Helm repository to fetch from (optional) - repository: - # repository url; supports exprimental oci helm fetch via - # oci:// scheme (required) - url: https://... - # specify helm binary version to use; - # '3' means binary 'helm3' needs to be on the path (optional) - helmVersion: '3' + path: github.com/cloudfoundry/cf-k8s-networking + helmChart: + # chart name (required) + name: stable/redis + # use specific chart version (string; optional) + version: '1.2.1' + # specifies Helm repository to fetch from (optional) + repository: + # repository url; supports exprimental oci helm fetch via + # oci:// scheme (required) + url: https://... + # specify helm binary version to use; + # '3' means binary 'helm3' needs to be on the path (optional) + helmVersion: '3' ``` From ef5d286e449b7dacb1014ac1ff6971f6420f6441 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 15:04:20 +0100 Subject: [PATCH 37/52] fix: remove extra import --- lib/modules/manager/vendir/extract.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 0d3df92fd547b7..81f935d266b51b 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -1,4 +1,3 @@ -import is from '@sindresorhus/is'; import { logger } from '../../../logger'; import { parseSingleYaml } from '../../../util/yaml'; import { HelmDatasource } from '../../datasource/helm'; From 478a9c902f72327333310890556c86e749c2fc7c Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 15:15:04 +0100 Subject: [PATCH 38/52] tests: alias test properly --- .../manager/vendir/__fixtures__/alias-contents.yaml | 2 +- lib/modules/manager/vendir/extract.spec.ts | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml index 9b0ae047d58b5b..fa7081a1c7e8e7 100644 --- a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml +++ b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml @@ -9,7 +9,7 @@ directories: name: oci-contour version: "7.10.1" repository: - url: alias:ociRegistry + url: oci://test - path: custom-repo-custom-version helmChart: name: normal-contour diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index fba13254e76f88..0e6c53705f242f 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -78,21 +78,20 @@ describe('modules/manager/vendir/extract', () => { }); it('resolves aliased registry urls', () => { - const ociresult = extractPackageFile(aliasContents, 'vendir.yml', { + const aliasResult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { placeholder: 'https://my-registry.gcr.io/', longAlias: 'https://registry.example.com/', - ociRegistry: 'oci://quay.example.com/organization', + 'oci://test': 'oci://quay.example.com/organization', }, }); - expect(ociresult).toMatchObject({ + expect(aliasResult).toMatchObject({ deps: [ { currentValue: '7.10.1', depName: 'oci-contour', - datasource: 'helm', - registryUrls: ['oci://quay.example.com/organization'], + datasource: 'docker', }, { currentValue: '7.10.1', From b7c8a1e6ed4f13f4ffbb7a111c44503767dcdd37 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 15:36:43 +0100 Subject: [PATCH 39/52] Update lib/modules/manager/vendir/extract.spec.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/extract.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 0e6c53705f242f..f5a0d8f0188e60 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -51,6 +51,7 @@ describe('modules/manager/vendir/extract', () => { { currentValue: '7.10.1', depName: 'contour', + packageName: 'charts.bitnami.com/bitnami/contour', datasource: 'docker', }, ], From a84203d92b8affea3b6caefc170e1ad682af1bf2 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 15:37:21 +0100 Subject: [PATCH 40/52] Update lib/modules/manager/vendir/extract.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/extract.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 81f935d266b51b..ddc7784783f371 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -58,18 +58,15 @@ export function parseVendir( content: string, packageFile?: string, ): VendirDefinition | null { - let pkg: VendirDefinition; try { - pkg = parseSingleYaml(content, { + return parseSingleYaml(content, { customSchema: Vendir, removeTemplates: true, }); - } catch (e) /* istanbul ignore next */ { + } catch (e) { logger.debug({ packageFile }, 'Error parsing vendir.yml file'); return null; } - - return pkg; } export function extractPackageFile( From 64b98548fa899ce4acdbdafbf20a57e9b886cefe Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 16:44:00 +0100 Subject: [PATCH 41/52] feat: add regex for validation on URL fix: remove unnecessary check fix: remove utils tests: update tests results to reflect better naming tests: inline emptydirectories --- .../vendir/__fixtures__/alias-contents.yaml | 12 +++++----- .../__fixtures__/empty-directories.yaml | 3 --- lib/modules/manager/vendir/extract.spec.ts | 24 +++++++++++++------ lib/modules/manager/vendir/extract.ts | 16 +++++-------- lib/modules/manager/vendir/schema.ts | 6 ++++- lib/modules/manager/vendir/utils.ts | 5 ---- 6 files changed, 34 insertions(+), 32 deletions(-) delete mode 100644 lib/modules/manager/vendir/__fixtures__/empty-directories.yaml delete mode 100644 lib/modules/manager/vendir/utils.ts diff --git a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml index fa7081a1c7e8e7..016d9acc5e8dac 100644 --- a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml +++ b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml @@ -6,25 +6,25 @@ directories: contents: - path: custom-repo-custom-version helmChart: - name: oci-contour + name: oci version: "7.10.1" repository: url: oci://test - path: custom-repo-custom-version helmChart: - name: normal-contour + name: https version: "7.10.1" repository: - url: alias:longAlias + url: alias:https://test - path: custom-repo-custom-version helmChart: - name: placeholder-contour + name: http version: "7.10.1" repository: - url: '@placeholder' + url: alias:http://test - path: custom-repo-custom-version helmChart: - name: broken-contour + name: broken version: "7.10.1" repository: url: alias:brokenAlias diff --git a/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml b/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml deleted file mode 100644 index b9e824668b4657..00000000000000 --- a/lib/modules/manager/vendir/__fixtures__/empty-directories.yaml +++ /dev/null @@ -1,3 +0,0 @@ -apiVersion: vendir.k14s.io/v1alpha1 -kind: Config -directories: [] diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index f5a0d8f0188e60..b23e3382ee9e5c 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { Fixtures } from '../../../../test/fixtures'; import { extractPackageFile } from '.'; @@ -6,7 +7,6 @@ const ociContents = Fixtures.get('oci-contents.yaml'); const aliasContents = Fixtures.get('alias-contents.yaml'); const multipleContents = Fixtures.get('multiple-contents.yaml'); const nonHelmChartContents = Fixtures.get('non-helmchart.yaml'); -const emptyDirectories = Fixtures.get('empty-directories.yaml'); describe('modules/manager/vendir/extract', () => { describe('extractPackageFile()', () => { @@ -21,6 +21,11 @@ describe('modules/manager/vendir/extract', () => { }); it('returns null for empty directories key', () => { + const emptyDirectories = codeBlock` + apiVersion: vendir.k14s.io/v1alpha1 + kind: Config + directories: [] + `; const result = extractPackageFile(emptyDirectories, 'vendir.yml', {}); expect(result).toBeNull(); }); @@ -81,8 +86,8 @@ describe('modules/manager/vendir/extract', () => { it('resolves aliased registry urls', () => { const aliasResult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { - placeholder: 'https://my-registry.gcr.io/', - longAlias: 'https://registry.example.com/', + 'http://test': 'https://my-registry.gcr.io/', + 'https://test': 'https://registry.example.com/', 'oci://test': 'oci://quay.example.com/organization', }, }); @@ -90,25 +95,30 @@ describe('modules/manager/vendir/extract', () => { expect(aliasResult).toMatchObject({ deps: [ { + currentDigest: undefined, currentValue: '7.10.1', - depName: 'oci-contour', + depName: 'oci', datasource: 'docker', + depType: 'HelmChart', + packageName: 'test/oci', + pinDigests: false, }, { currentValue: '7.10.1', - depName: 'normal-contour', + depName: 'https', datasource: 'helm', + depType: 'HelmChart', registryUrls: ['https://registry.example.com/'], }, { currentValue: '7.10.1', - depName: 'placeholder-contour', + depName: 'http', datasource: 'helm', registryUrls: ['https://my-registry.gcr.io/'], }, { currentValue: '7.10.1', - depName: 'broken-contour', + depName: 'broken', datasource: 'helm', skipReason: 'placeholder-url', }, diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index ddc7784783f371..1311defa069ed2 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -10,8 +10,6 @@ import type { } from '../types'; import { HelmChartDefinition, Vendir, VendirDefinition } from './schema'; -import { isHelmChart } from './utils'; - // TODO: Add support for other vendir types (like git tags, github releases, etc.) // Recommend looking at the kustomize manager for more information on support. @@ -85,14 +83,12 @@ export function extractPackageFile( // grab the helm charts const contents = pkg.directories.flatMap((directory) => directory.contents); for (const content of contents) { - if (isHelmChart(content)) { - const dep = extractHelmChart(content.helmChart, config.registryAliases); - if (dep) { - deps.push({ - ...dep, - depType: 'HelmChart', - }); - } + const dep = extractHelmChart(content.helmChart, config.registryAliases); + if (dep) { + deps.push({ + ...dep, + depType: 'HelmChart', + }); } } diff --git a/lib/modules/manager/vendir/schema.ts b/lib/modules/manager/vendir/schema.ts index 01472e1403ba35..2a726c32afd03f 100644 --- a/lib/modules/manager/vendir/schema.ts +++ b/lib/modules/manager/vendir/schema.ts @@ -10,7 +10,11 @@ export const HelmChart = z.object({ name: z.string(), version: z.string(), repository: z.object({ - url: z.string(), + // Because helm charts support multiple protocols https and oci + // We need to check for the following. Further, because we use the helmv3 manager + // to handle registryAliases for http or https sources we need to support alias and @ + // as prefixes + url: z.string().regex(/^(?:oci:|https?:|alias:|@).+/), }), }); diff --git a/lib/modules/manager/vendir/utils.ts b/lib/modules/manager/vendir/utils.ts deleted file mode 100644 index d721141571d18f..00000000000000 --- a/lib/modules/manager/vendir/utils.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { Contents, HelmChartContent } from './types'; - -export function isHelmChart(item: Contents): item is HelmChartContent { - return 'helmChart' in item; -} From 21f3c3859a6626d7a242531e384ea1d7601e6299 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 16:50:21 +0100 Subject: [PATCH 42/52] docs: add info to readme on aliases --- lib/modules/manager/vendir/readme.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index de5b19c17ad2e1..2a0272d84a5a7a 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -26,3 +26,15 @@ directories: # '3' means binary 'helm3' needs to be on the path (optional) helmVersion: '3' ``` + +## Registry Aliases + +The Vendir Manager supports registry aliases for helm charts for both OCI and HTTP/HTTPS registries. + +### HTTP/HTTPS + +Aliases for http/https should be prefixed with: `alias` or `@` similar to the Helmv3 manager + +### OCI + +Aliases for OCI registries are supported via the dockerfile/docker manager From 63af3298403e5adc56a3a9879bff6d07618d32f8 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 17:46:35 +0100 Subject: [PATCH 43/52] fix: remove aliases from http/https as it isn't needed --- .../vendir/__fixtures__/alias-contents.yaml | 18 ---------------- lib/modules/manager/vendir/extract.spec.ts | 21 ------------------- lib/modules/manager/vendir/extract.ts | 16 ++------------ lib/modules/manager/vendir/readme.md | 6 ------ lib/modules/manager/vendir/schema.ts | 6 +----- 5 files changed, 3 insertions(+), 64 deletions(-) diff --git a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml index 016d9acc5e8dac..1090312b61a6de 100644 --- a/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml +++ b/lib/modules/manager/vendir/__fixtures__/alias-contents.yaml @@ -10,21 +10,3 @@ directories: version: "7.10.1" repository: url: oci://test - - path: custom-repo-custom-version - helmChart: - name: https - version: "7.10.1" - repository: - url: alias:https://test - - path: custom-repo-custom-version - helmChart: - name: http - version: "7.10.1" - repository: - url: alias:http://test - - path: custom-repo-custom-version - helmChart: - name: broken - version: "7.10.1" - repository: - url: alias:brokenAlias diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index b23e3382ee9e5c..b88ffd0e792219 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -86,8 +86,6 @@ describe('modules/manager/vendir/extract', () => { it('resolves aliased registry urls', () => { const aliasResult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { - 'http://test': 'https://my-registry.gcr.io/', - 'https://test': 'https://registry.example.com/', 'oci://test': 'oci://quay.example.com/organization', }, }); @@ -103,25 +101,6 @@ describe('modules/manager/vendir/extract', () => { packageName: 'test/oci', pinDigests: false, }, - { - currentValue: '7.10.1', - depName: 'https', - datasource: 'helm', - depType: 'HelmChart', - registryUrls: ['https://registry.example.com/'], - }, - { - currentValue: '7.10.1', - depName: 'http', - datasource: 'helm', - registryUrls: ['https://my-registry.gcr.io/'], - }, - { - currentValue: '7.10.1', - depName: 'broken', - datasource: 'helm', - skipReason: 'placeholder-url', - }, ], }); }); diff --git a/lib/modules/manager/vendir/extract.ts b/lib/modules/manager/vendir/extract.ts index 1311defa069ed2..94049a52ae628b 100644 --- a/lib/modules/manager/vendir/extract.ts +++ b/lib/modules/manager/vendir/extract.ts @@ -2,7 +2,7 @@ import { logger } from '../../../logger'; import { parseSingleYaml } from '../../../util/yaml'; import { HelmDatasource } from '../../datasource/helm'; import { getDep } from '../dockerfile/extract'; -import { isOCIRegistry, resolveAlias } from '../helmv3/utils'; +import { isOCIRegistry } from '../helmv3/utils'; import type { ExtractConfig, PackageDependency, @@ -32,22 +32,10 @@ export function extractHelmChart( pinDigests: false, }; } - let repository = helmChart.repository.url || null; - if (aliases) { - repository = resolveAlias(helmChart.repository.url, aliases); - } - if (!repository) { - return { - depName: helmChart.name, - currentValue: helmChart.version, - datasource: HelmDatasource.id, - skipReason: 'placeholder-url', - }; - } return { depName: helmChart.name, currentValue: helmChart.version, - registryUrls: [repository], + registryUrls: [helmChart.repository.url], datasource: HelmDatasource.id, }; } diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 2a0272d84a5a7a..8cc5ee34c97e7c 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -29,12 +29,6 @@ directories: ## Registry Aliases -The Vendir Manager supports registry aliases for helm charts for both OCI and HTTP/HTTPS registries. - -### HTTP/HTTPS - -Aliases for http/https should be prefixed with: `alias` or `@` similar to the Helmv3 manager - ### OCI Aliases for OCI registries are supported via the dockerfile/docker manager diff --git a/lib/modules/manager/vendir/schema.ts b/lib/modules/manager/vendir/schema.ts index 2a726c32afd03f..8e761a450a01e7 100644 --- a/lib/modules/manager/vendir/schema.ts +++ b/lib/modules/manager/vendir/schema.ts @@ -10,11 +10,7 @@ export const HelmChart = z.object({ name: z.string(), version: z.string(), repository: z.object({ - // Because helm charts support multiple protocols https and oci - // We need to check for the following. Further, because we use the helmv3 manager - // to handle registryAliases for http or https sources we need to support alias and @ - // as prefixes - url: z.string().regex(/^(?:oci:|https?:|alias:|@).+/), + url: z.string().regex(/^(?:oci|https?):\/\/.+/), }), }); From 2dc9a1b2fd074a7fa05d8b2e72823bf7fc3914e2 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Wed, 20 Mar 2024 18:17:58 +0100 Subject: [PATCH 44/52] fix: test pass --- lib/modules/manager/vendir/artifacts.ts | 4 ++-- lib/modules/manager/vendir/readme.md | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index f91bb311f472f9..492acc4fa9d6df 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -70,7 +70,7 @@ export async function updateArtifacts({ const status = await getRepoStatus(); if (status) { const modifiedFiles = status.modified ?? []; - const notAddedFiles = status.not_added ?? []; + const notAddedFiles = status.not_added; const deletedFiles = status.deleted ?? []; for (const f of modifiedFiles.concat(notAddedFiles)) { @@ -98,7 +98,7 @@ export async function updateArtifacts({ logger.error('Failed to get git status'); } - return fileChanges.length > 0 ? fileChanges : null; + return fileChanges; } catch (err) { if (err.message === TEMPORARY_ERROR) { throw err; diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 8cc5ee34c97e7c..2283c29833f00e 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -1,3 +1,5 @@ +# Vendir + Renovate supports updating Helm Chart references in vendir.yml via the [vendir](https://carvel.dev/vendir/) tool. Renovate requires the presence of a [vendir lock file](https://carvel.dev/vendir/docs/v0.40.x/vendir-lock-spec/) which is generated by vendir and should be stored in source code. It supports both https and oci helm chart repositories. From 91d094eb90480292a4772d474b5f36ff45003d5a Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 10:51:18 +0100 Subject: [PATCH 45/52] docs: remove # line --- lib/modules/manager/vendir/readme.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 2283c29833f00e..8cc5ee34c97e7c 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -1,5 +1,3 @@ -# Vendir - Renovate supports updating Helm Chart references in vendir.yml via the [vendir](https://carvel.dev/vendir/) tool. Renovate requires the presence of a [vendir lock file](https://carvel.dev/vendir/docs/v0.40.x/vendir-lock-spec/) which is generated by vendir and should be stored in source code. It supports both https and oci helm chart repositories. From 1cbe927bbaf54688def7a81bad4bfc5f63ee148d Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 14:48:46 +0100 Subject: [PATCH 46/52] Update lib/modules/manager/vendir/readme.md Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 8cc5ee34c97e7c..5d7c840415a753 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -29,6 +29,6 @@ directories: ## Registry Aliases -### OCI +#### OCI Aliases for OCI registries are supported via the dockerfile/docker manager From c56a31229ed60d1e32ef740c01447b7a979d9bfe Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 14:48:56 +0100 Subject: [PATCH 47/52] Update lib/modules/manager/vendir/readme.md Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/readme.md b/lib/modules/manager/vendir/readme.md index 5d7c840415a753..3c7eaf0f97e591 100644 --- a/lib/modules/manager/vendir/readme.md +++ b/lib/modules/manager/vendir/readme.md @@ -27,7 +27,7 @@ directories: helmVersion: '3' ``` -## Registry Aliases +### Registry Aliases #### OCI From 4e3912314294976198e858c272599997307f6b0c Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 14:49:41 +0100 Subject: [PATCH 48/52] Update lib/modules/manager/vendir/extract.spec.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/extract.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index b88ffd0e792219..9c65f89882fc53 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -86,7 +86,7 @@ describe('modules/manager/vendir/extract', () => { it('resolves aliased registry urls', () => { const aliasResult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { - 'oci://test': 'oci://quay.example.com/organization', + 'test': 'quay.example.com/organization', }, }); From 45d432ebb3647d84ea0db1df429bf9cdca9cf4b8 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 14:49:50 +0100 Subject: [PATCH 49/52] Update lib/modules/manager/vendir/extract.spec.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/extract.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 9c65f89882fc53..0e1016b4c1d8b6 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -98,7 +98,7 @@ describe('modules/manager/vendir/extract', () => { depName: 'oci', datasource: 'docker', depType: 'HelmChart', - packageName: 'test/oci', + packageName: 'quay.example.com/organization/oci', pinDigests: false, }, ], From e57211040a48770c1786f5158046f034758d1a39 Mon Sep 17 00:00:00 2001 From: Hans Knecht Date: Thu, 21 Mar 2024 14:50:40 +0100 Subject: [PATCH 50/52] Update lib/modules/manager/vendir/artifacts.ts Co-authored-by: Michael Kriese --- lib/modules/manager/vendir/artifacts.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 492acc4fa9d6df..14380b7fe04d99 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -98,7 +98,7 @@ export async function updateArtifacts({ logger.error('Failed to get git status'); } - return fileChanges; + return fileChanges.length ? fileChanges: null; } catch (err) { if (err.message === TEMPORARY_ERROR) { throw err; From a4b893386da25a626c3de9386ab9c9bf7e137747 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 21 Mar 2024 16:37:56 +0100 Subject: [PATCH 51/52] Drop unnecessary branch --- lib/modules/manager/vendir/artifacts.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/modules/manager/vendir/artifacts.ts b/lib/modules/manager/vendir/artifacts.ts index 14380b7fe04d99..17db506dd5d7b9 100644 --- a/lib/modules/manager/vendir/artifacts.ts +++ b/lib/modules/manager/vendir/artifacts.ts @@ -58,9 +58,6 @@ export async function updateArtifacts({ contents: newVendirLockContent, }, }); - } else { - logger.debug('vendir.lock.yml is unchanged'); - return null; } // add modified vendir archives to artifacts @@ -98,7 +95,7 @@ export async function updateArtifacts({ logger.error('Failed to get git status'); } - return fileChanges.length ? fileChanges: null; + return fileChanges.length ? fileChanges : null; } catch (err) { if (err.message === TEMPORARY_ERROR) { throw err; From cda1c74c73eb46b956557d89fa72982e5c4db43c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Reegn?= Date: Thu, 21 Mar 2024 16:40:37 +0100 Subject: [PATCH 52/52] Fix lint issue --- lib/modules/manager/vendir/extract.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/vendir/extract.spec.ts b/lib/modules/manager/vendir/extract.spec.ts index 0e1016b4c1d8b6..a4d612c47322f3 100644 --- a/lib/modules/manager/vendir/extract.spec.ts +++ b/lib/modules/manager/vendir/extract.spec.ts @@ -86,7 +86,7 @@ describe('modules/manager/vendir/extract', () => { it('resolves aliased registry urls', () => { const aliasResult = extractPackageFile(aliasContents, 'vendir.yml', { registryAliases: { - 'test': 'quay.example.com/organization', + test: 'quay.example.com/organization', }, });