diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5a133fe51d6b2b..e36de07977df81 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM containerbase/node:14.17.5@sha256:3fce5dfc917107aca3cf3823a42ffae92a4bb433a55aa1f5be5f7caeb0019b20 +FROM containerbase/node:14.17.6@sha256:ebe928dcb70c9ec5ad36adc576218d81a3f2a598f0b8cfa14c23467957596389 # renovate: datasource=npm diff --git a/.vscode/launch.json b/.vscode/launch.json index a8802c8f3c56a2..cedef1459c4c08 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,6 +22,7 @@ "args": [ "--runInBand", "--collectCoverage=false", + "--testTimeout 100000000", "--runTestsByPath", "${relativeFile}" ], @@ -42,7 +43,11 @@ "request": "launch", "name": "Jest All", "program": "${workspaceFolder}/node_modules/.bin/jest", - "args": ["--runInBand", "--collectCoverage=false"], + "args": [ + "--runInBand", + "--collectCoverage=false", + "--testTimeout 100000000" + ], "env": { "NODE_ENV": "test", "LOG_LEVEL": "debug" @@ -64,7 +69,12 @@ "disableOptimisticBPs": true, "cwd": "${workspaceFolder}", "runtimeExecutable": "yarn", - "args": ["jest", "--runInBand", "--watchAll=false"] + "args": [ + "jest", + "--runInBand", + "--watchAll=false", + "--testTimeout 100000000" + ] } ] } diff --git a/docs/usage/automerge-configuration.md b/docs/usage/key-concepts/automerge.md similarity index 100% rename from docs/usage/automerge-configuration.md rename to docs/usage/key-concepts/automerge.md diff --git a/lib/config/__snapshots__/massage.spec.ts.snap b/lib/config/__snapshots__/massage.spec.ts.snap index f4604219f2e146..9ad140af073156 100644 --- a/lib/config/__snapshots__/massage.spec.ts.snap +++ b/lib/config/__snapshots__/massage.spec.ts.snap @@ -1,5 +1,26 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`config/massage massageConfig does not massage lockFileMaintenance 1`] = ` +Object { + "packageRules": Array [ + Object { + "lockFileMaintenance": Object { + "enabled": true, + }, + "matchBaseBranches": Array [ + "release/ft10/1.9.x", + ], + "matchManagers": Array [ + "helmv3", + ], + "schedule": Array [ + "at any time", + ], + }, + ], +} +`; + exports[`config/massage massageConfig massages packageRules matchUpdateTypes 1`] = ` Object { "packageRules": Array [ diff --git a/lib/config/massage.spec.ts b/lib/config/massage.spec.ts index d952354cd8bbe3..3b1c7b2a777db5 100644 --- a/lib/config/massage.spec.ts +++ b/lib/config/massage.spec.ts @@ -42,5 +42,20 @@ describe('config/massage', () => { expect(res).toMatchSnapshot(); expect(res.packageRules).toHaveLength(3); }); + it('does not massage lockFileMaintenance', () => { + const config: RenovateConfig = { + packageRules: [ + { + matchManagers: ['helmv3'], + matchBaseBranches: ['release/ft10/1.9.x'], + lockFileMaintenance: { enabled: true }, + schedule: ['at any time'], + }, + ], + }; + const res = massage.massageConfig(config); + expect(res).toMatchSnapshot(); + expect(res.packageRules).toHaveLength(1); + }); }); }); diff --git a/lib/config/massage.ts b/lib/config/massage.ts index f8e0bc7aa7c87c..1b7eb50f58119e 100644 --- a/lib/config/massage.ts +++ b/lib/config/massage.ts @@ -47,7 +47,6 @@ export function massageConfig(config: RenovateConfig): RenovateConfig { 'patch', 'pin', 'digest', - 'lockFileMaintenance', 'rollback', ]; for (const rule of massagedConfig.packageRules) { diff --git a/lib/datasource/api.ts b/lib/datasource/api.ts index b6c3d511372139..356af4bc8a106f 100644 --- a/lib/datasource/api.ts +++ b/lib/datasource/api.ts @@ -11,6 +11,7 @@ import * as gitRefs from './git-refs'; import * as gitTags from './git-tags'; import * as githubReleases from './github-releases'; import * as githubTags from './github-tags'; +import { GitlabReleasesDatasource } from './gitlab-releases'; import * as gitlabTags from './gitlab-tags'; import * as go from './go'; import { GradleVersionDatasource } from './gradle-version'; @@ -50,6 +51,7 @@ api.set('git-tags', gitTags); api.set('github-releases', githubReleases); api.set('github-tags', githubTags); api.set('gitlab-tags', gitlabTags); +api.set(GitlabReleasesDatasource.id, new GitlabReleasesDatasource()); api.set('go', go); api.set('gradle-version', new GradleVersionDatasource()); api.set('helm', new HelmDatasource()); diff --git a/lib/datasource/docker/__snapshots__/index.spec.ts.snap b/lib/datasource/docker/__snapshots__/index.spec.ts.snap index 461f44c1921e19..8cdb641738aefc 100644 --- a/lib/datasource/docker/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/docker/__snapshots__/index.spec.ts.snap @@ -1,778 +1,22 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`datasource/docker/index getDigest continues without token if ECR authentication could not be extracted 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, -] -`; - -exports[`datasource/docker/index getDigest continues without token if ECR authentication fails 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, -] -`; - -exports[`datasource/docker/index getDigest continues without token, when no header is present 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-new-value", - }, -] -`; - -exports[`datasource/docker/index getDigest falls back to body for digest 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "auth.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/some-dep:pull", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-new-value", - }, -] -`; - -exports[`datasource/docker/index getDigest passes credentials to ECR client 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic abcdef", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic abcdef", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/manifests/some-tag", - }, -] -`; - -exports[`datasource/docker/index getDigest returns digest 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "host": "auth.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/some-dep:pull", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/latest", - }, -] -`; - -exports[`datasource/docker/index getDigest returns null for 403 with basic authentication 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, -] -`; - -exports[`datasource/docker/index getDigest returns null if empty header 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-new-value", - }, -] -`; - -exports[`datasource/docker/index getDigest returns null if errored 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-new-value", - }, -] -`; - -exports[`datasource/docker/index getDigest returns null if no token 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-new-value", - }, -] -`; - -exports[`datasource/docker/index getDigest supports ECR authentication 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic abcdef", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic abcdef", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/manifests/some-tag", - }, -] -`; - -exports[`datasource/docker/index getDigest supports basic authentication 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-dep/manifests/some-tag", - }, -] -`; - -exports[`datasource/docker/index getDigest supports docker insecure registry 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "http://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "http://index.docker.io/v2/library/some-dep/manifests/latest", - }, -] -`; - -exports[`datasource/docker/index getDigest supports scoped names 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "auth.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/some-other-dep:pull", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/some-other-dep/manifests/8.0.0-alpine", - }, -] -`; - -exports[`datasource/docker/index getReleases adds library/ prefix for Docker Hub (explicit) 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "auth.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/node:pull", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token ", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/node/manifests/1.0.0", - }, -] -`; - -exports[`datasource/docker/index getReleases adds library/ prefix for Docker Hub (implicit) 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "auth.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://auth.docker.io/token?service=registry.docker.io&scope=repository:library/node:pull", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token ", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/node/manifests/1.0.0", - }, -] -`; - -exports[`datasource/docker/index getReleases adds no library/ prefix for other registries 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "k8s.gcr.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://k8s.gcr.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "k8s.gcr.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://k8s.gcr.io/v2/token?service=k8s.gcr.io&scope=repository:kubernetes-dashboard-amd64:pull", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Bearer some-token ", - "host": "k8s.gcr.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://k8s.gcr.io/v2/kubernetes-dashboard-amd64/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "k8s.gcr.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://k8s.gcr.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "host": "k8s.gcr.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://k8s.gcr.io/v2/kubernetes-dashboard-amd64/manifests/1.0.0", - }, -] -`; - exports[`datasource/docker/index getReleases ignores unsupported manifest 1`] = ` Object { - "registryUrl": "https://index.docker.io", + "registryUrl": "https://registry.company.com", "releases": Array [], } `; -exports[`datasource/docker/index getReleases ignores unsupported manifest 2`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/latest", - }, -] -`; - exports[`datasource/docker/index getReleases ignores unsupported schema version 1`] = ` Object { - "registryUrl": "https://index.docker.io", + "registryUrl": "https://registry.company.com", "releases": Array [], } `; -exports[`datasource/docker/index getReleases ignores unsupported schema version 2`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/latest", - }, -] -`; - -exports[`datasource/docker/index getReleases returns null if no auth 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, -] -`; - -exports[`datasource/docker/index getReleases returns null if no token 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/library/node/tags/list?n=10000", - }, -] -`; - -exports[`datasource/docker/index getReleases returns null on error 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "index.docker.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://index.docker.io/v2/my/node/tags/list?n=10000", - }, -] -`; - exports[`datasource/docker/index getReleases supports labels 1`] = ` Object { - "registryUrl": "https://index.docker.io", + "registryUrl": "https://registry.company.com", "releases": Array [ Object { "version": "1.0.0", @@ -797,437 +41,17 @@ Object { } `; -exports[`datasource/docker/index getReleases supports labels 2`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/2-alpine", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/blobs/some-config-digest", - }, -] -`; - exports[`datasource/docker/index getReleases supports manifest lists 1`] = ` Object { - "registryUrl": "https://index.docker.io", + "registryUrl": "https://registry.company.com", "releases": Array [], "sourceUrl": "https://github.com/renovatebot/renovate", } `; -exports[`datasource/docker/index getReleases supports manifest lists 2`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/abc", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/some-image-digest", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/blobs/some-config-digest", - }, -] -`; - exports[`datasource/docker/index getReleases supports redirect 1`] = ` Object { - "registryUrl": "https://index.docker.io", + "registryUrl": "https://registry.company.com", "releases": Array [], } `; - -exports[`datasource/docker/index getReleases supports redirect 2`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/latest", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/blobs/some-config-digest", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "host": "abc.s3.amazon.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://abc.s3.amazon.com/some-config-digest?X-Amz-Algorithm=xxxx", - }, -] -`; - -exports[`datasource/docker/index getReleases uses custom registry in depName 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/1.0.0", - }, -] -`; - -exports[`datasource/docker/index getReleases uses custom registry with registryUrls 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/tags/list?n=10000", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "api.github.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://api.github.com/user/9287/repos?page=3&per_page=100", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "registry.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://registry.company.com/v2/node/manifests/latest", - }, -] -`; - -exports[`datasource/docker/index getReleases uses lower tag limit for ECR deps 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/tags/list?n=1000", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "123456789.dkr.ecr.us-east-1.amazonaws.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://123456789.dkr.ecr.us-east-1.amazonaws.com/v2/node/manifests/undefined", - }, -] -`; - -exports[`datasource/docker/index getReleases uses quay api 1`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "quay.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://quay.io/api/v1/repository/bitnami/redis/tag/?limit=100&page=1&onlyActiveTags=true", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "quay.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://quay.io/v2/", - }, - Object { - "headers": Object { - "accept": "application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json", - "accept-encoding": "gzip, deflate, br", - "authorization": "Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk", - "host": "quay.io", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://quay.io/v2/bitnami/redis/manifests/5.0.12", - }, -] -`; diff --git a/lib/datasource/docker/common.spec.ts b/lib/datasource/docker/common.spec.ts index d6d6e4d1e331cd..37d8c922f0992a 100644 --- a/lib/datasource/docker/common.spec.ts +++ b/lib/datasource/docker/common.spec.ts @@ -75,7 +75,7 @@ describe('datasource/docker/common', () => { beforeEach(() => { httpMock .scope('https://my.local.registry') - .get('/v2/') + .get('/v2/', undefined, { badheaders: ['authorization'] }) .reply(401, '', { 'www-authenticate': 'Authenticate you must' }); hostRules.hosts.mockReturnValue([]); }); @@ -92,10 +92,10 @@ describe('datasource/docker/common', () => { ); expect(headers).toMatchInlineSnapshot(` - Object { - "authorization": "some-authType some-token", - } - `); +Object { + "authorization": "some-authType some-token", +} +`); }); it('returns "Bearer token" if only token provided', async () => { @@ -109,10 +109,26 @@ describe('datasource/docker/common', () => { ); expect(headers).toMatchInlineSnapshot(` - Object { - "authorization": "Bearer some-token", - } - `); +Object { + "authorization": "Bearer some-token", +} +`); + }); + + it('fails', async () => { + httpMock.clear(false); + + httpMock + .scope('https://my.local.registry') + .get('/v2/', undefined, { badheaders: ['authorization'] }) + .reply(401, '', {}); + + const headers = await dockerCommon.getAuthHeaders( + 'https://my.local.registry', + 'https://my.local.registry/prefix' + ); + + expect(headers).toBeNull(); }); }); }); diff --git a/lib/datasource/docker/common.ts b/lib/datasource/docker/common.ts index f50a1ae99dd456..3b101a6a254c6f 100644 --- a/lib/datasource/docker/common.ts +++ b/lib/datasource/docker/common.ts @@ -1,4 +1,5 @@ import { ECR, ECRClientConfig } from '@aws-sdk/client-ecr'; +import is from '@sindresorhus/is'; import hasha from 'hasha'; import wwwAuthenticate from 'www-authenticate'; import { HOST_DISABLED } from '../../constants/error-messages'; @@ -7,7 +8,7 @@ import { HostRule } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import * as hostRules from '../../util/host-rules'; -import { Http, HttpResponse } from '../../util/http'; +import { Http, HttpOptions, HttpResponse } from '../../util/http'; import type { OutgoingHttpHeaders } from '../../util/http/types'; import { ensureTrailingSlash, @@ -59,24 +60,47 @@ export async function getAuthHeaders( const apiCheckUrl = `${registryHost}/v2/`; const apiCheckResponse = await http.get(apiCheckUrl, { throwHttpErrors: false, + noAuth: true, }); - if (apiCheckResponse.headers['www-authenticate'] === undefined) { + + if (apiCheckResponse.statusCode === 200) { + logger.debug({ registryHost }, 'No registry auth required'); return {}; } + if ( + apiCheckResponse.statusCode !== 401 || + !is.nonEmptyString(apiCheckResponse.headers['www-authenticate']) + ) { + logger.warn( + { registryHost, res: apiCheckResponse }, + 'Invalid registry response' + ); + return null; + } + const authenticateHeader = new wwwAuthenticate.parsers.WWW_Authenticate( apiCheckResponse.headers['www-authenticate'] ); - const opts: HostRule & { - headers?: Record; - } = hostRules.find({ hostType: id, url: apiCheckUrl }); + const opts: HostRule & HttpOptions = hostRules.find({ + hostType: id, + url: apiCheckUrl, + }); if (ecrRegex.test(registryHost)) { + logger.trace( + { registryHost, dockerRepository }, + `Using ecr auth for Docker registry` + ); const [, region] = ecrRegex.exec(registryHost); const auth = await getECRAuthToken(region, opts); if (auth) { opts.headers = { authorization: `Basic ${auth}` }; } } else if (opts.username && opts.password) { + logger.trace( + { registryHost, dockerRepository }, + `Using basic auth for Docker registry` + ); const auth = Buffer.from(`${opts.username}:${opts.password}`).toString( 'base64' ); @@ -84,26 +108,38 @@ export async function getAuthHeaders( } else if (opts.token) { const authType = opts.authType ?? 'Bearer'; logger.trace( - `Using ${authType} token for Docker registry ${registryHost}` + { registryHost, dockerRepository }, + `Using ${authType} token for Docker registry` ); opts.headers = { authorization: `${authType} ${opts.token}` }; - return opts.headers; } delete opts.username; delete opts.password; delete opts.token; - if (authenticateHeader.scheme.toUpperCase() === 'BASIC') { - logger.trace(`Using Basic auth for docker registry ${registryHost}`); - await http.get(apiCheckUrl, opts); + // If realm isn't an url, we should directly use auth header + // Can happen when we get a Basic auth or some other auth type + // * WWW-Authenticate: Basic realm="Artifactory Realm" + // * Www-Authenticate: Basic realm="https://123456789.dkr.ecr.eu-central-1.amazonaws.com/",service="ecr.amazonaws.com" + // * www-authenticate: Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:user/image:pull" + // * www-authenticate: Bearer realm="https://auth.docker.io/token",service="registry.docker.io" + if ( + authenticateHeader.scheme.toUpperCase() !== 'BEARER' || + parseUrl(authenticateHeader.parms.realm) == null + ) { + logger.trace( + { registryHost, dockerRepository, authenticateHeader }, + `Invalid realm, testing direct auth` + ); return opts.headers; } - // prettier-ignore - const authUrl = `${String(authenticateHeader.parms.realm)}?service=${String(authenticateHeader.parms.service)}&scope=repository:${dockerRepository}:pull`; + const authUrl = `${authenticateHeader.parms.realm}?service=${authenticateHeader.parms.service}&scope=repository:${dockerRepository}:pull`; logger.trace( - `Obtaining docker registry token for ${dockerRepository} using url ${authUrl}` + { registryHost, dockerRepository, authUrl }, + `Obtaining docker registry token` ); + opts.noAuth = true; const authResponse = ( await http.getJson<{ token?: string; access_token?: string }>( authUrl, @@ -224,20 +260,18 @@ function digestFromManifestStr(str: hasha.HashaInput): string { return 'sha256:' + hasha(str, { algorithm: 'sha256' }); } -export function extractDigestFromResponse( +export function extractDigestFromResponseBody( manifestResponse: HttpResponse ): string { - if (manifestResponse.headers['docker-content-digest'] === undefined) { - return digestFromManifestStr(manifestResponse.body); - } - return manifestResponse.headers['docker-content-digest'] as string; + return digestFromManifestStr(manifestResponse.body); } // TODO: debug why quay throws errors (#9612) export async function getManifestResponse( registryHost: string, dockerRepository: string, - tag: string + tag: string, + mode: 'head' | 'get' = 'get' ): Promise { logger.debug( `getManifestResponse(${registryHost}, ${dockerRepository}, ${tag})` @@ -251,8 +285,9 @@ export async function getManifestResponse( headers.accept = 'application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json'; const url = `${registryHost}/v2/${dockerRepository}/manifests/${tag}`; - const manifestResponse = await http.get(url, { + const manifestResponse = await http[mode](url, { headers, + noAuth: true, }); return manifestResponse; } catch (err) /* istanbul ignore next */ { @@ -399,6 +434,7 @@ export async function getLabels( const url = `${registryHost}/v2/${dockerRepository}/blobs/${configDigest}`; const configResponse = await http.get(url, { headers, + noAuth: true, }); labels = JSON.parse(configResponse.body).config.Labels; diff --git a/lib/datasource/docker/index.spec.ts b/lib/datasource/docker/index.spec.ts index 7ed91d93120bf9..33cd9df8d4e1a3 100644 --- a/lib/datasource/docker/index.spec.ts +++ b/lib/datasource/docker/index.spec.ts @@ -59,54 +59,58 @@ describe('datasource/docker/index', () => { it('returns null if no token', async () => { httpMock .scope(baseUrl) - .get('/') + .get('/', undefined, { badheaders: ['authorization'] }) .reply(200, '', {}) - .get('/library/some-dep/manifests/some-new-value') + .head('/library/some-dep/manifests/some-new-value', undefined, { + badheaders: ['authorization'], + }) .reply(401); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-new-value' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('returns null if errored', async () => { httpMock .scope(baseUrl) - .get('/') - .reply(200, { token: 'some-token' }) - .get('/library/some-dep/manifests/some-new-value') + .get('/', undefined, { badheaders: ['authorization'] }) + .reply(200, { token: 'abc' }) + .head('/library/some-dep/manifests/some-new-value', undefined, { + reqheaders: { authorization: 'Bearer abc' }, + }) .replyWithError('error'); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-new-value' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('returns null if empty header', async () => { httpMock .scope(baseUrl) - .get('/') + .get('/', undefined, { badheaders: ['authorization'] }) .reply(200, { token: 'some-token' }) - .get('/library/some-dep/manifests/some-new-value') + .head('/library/some-dep/manifests/some-new-value') .reply(200, undefined, { 'docker-content-digest': '' }); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-new-value' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('returns digest', async () => { httpMock .scope(baseUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull "', }) - .get('/library/some-dep/manifests/latest') + .head('/library/some-dep/manifests/latest') .reply(200, {}, { 'docker-content-digest': 'some-digest' }); httpMock .scope(authUrl) @@ -121,16 +125,19 @@ describe('datasource/docker/index', () => { depName: 'some-dep', }); expect(res).toBe('some-digest'); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('falls back to body for digest', async () => { httpMock .scope(baseUrl) .get('/') - .reply(200, '', { + .twice() + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull "', }) + .head('/library/some-dep/manifests/some-new-value') + .reply(200, undefined, {}) .get('/library/some-dep/manifests/some-new-value') .reply( 200, @@ -161,6 +168,7 @@ describe('datasource/docker/index', () => { .get( '/token?service=registry.docker.io&scope=repository:library/some-dep:pull' ) + .twice() .reply(200, { token: 'some-token' }); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, @@ -169,14 +177,14 @@ describe('datasource/docker/index', () => { expect(res).toBe( 'sha256:b3d6068234f3a18ebeedd2dab81e67b6a192e81192a099df4112ecfc7c3be84f' ); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('supports docker insecure registry', async () => { httpMock .scope(baseUrl.replace('https', 'http')) - .get('/') - .reply(200, '', {}) - .get('/library/some-dep/manifests/latest') + .get('/', undefined, { badheaders: ['authorization'] }) + .reply(200) + .head('/library/some-dep/manifests/latest') .reply(200, '', { 'docker-content-digest': 'some-digest' }); hostRules.find.mockReturnValueOnce({ insecureRegistry: true }); const res = await getDigest({ @@ -184,56 +192,54 @@ describe('datasource/docker/index', () => { depName: 'some-dep', }); expect(res).toBe('some-digest'); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('supports basic authentication', async () => { httpMock .scope(baseUrl) - .get('/') - .reply(200, '', { + .get('/', undefined, { badheaders: ['authorization'] }) + .reply(401, '', { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }) - .get('/') - .reply(200) - .get('/library/some-dep/manifests/some-tag') + .head('/library/some-dep/manifests/some-tag', undefined, { + reqheaders: { + authorization: 'Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk', + }, + }) .reply(200, '', { 'docker-content-digest': 'some-digest' }); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-tag' ); - const trace = httpMock.getTrace(); expect(res).toBe('some-digest'); - expect(trace[1].headers.authorization).toBe( - 'Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk' - ); - expect(trace).toMatchSnapshot(); }); + it('returns null for 403 with basic authentication', async () => { httpMock .scope(baseUrl) - .get('/') - .reply(200, '', { + .get('/', undefined, { badheaders: ['authorization'] }) + .reply(401, '', { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }) - .get('/') + .head('/library/some-dep/manifests/some-tag') .reply(403); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-tag' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('passes credentials to ECR client', async () => { httpMock .scope(amazonUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }) - .get('/') - .reply(200) - .get('/node/manifests/some-tag') + .head('/node/manifests/some-tag', undefined, { + reqheaders: { authorization: 'Basic abc' }, + }) .reply(200, '', { 'docker-content-digest': 'some-digest' }); mockEcrAuthResolve({ @@ -248,8 +254,6 @@ describe('datasource/docker/index', () => { 'some-tag' ); - const trace = httpMock.getTrace(); - expect(trace).toMatchSnapshot(); expect(AWS.ECR).toHaveBeenCalledWith({ credentials: { accessKeyId: 'some-username', @@ -258,20 +262,21 @@ describe('datasource/docker/index', () => { region: 'us-east-1', }); }); + it('supports ECR authentication', async () => { httpMock .scope(amazonUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', }) - .get('/') - .reply(200) - .get('/node/manifests/some-tag') + .head('/node/manifests/some-tag', undefined, { + reqheaders: { authorization: 'Basic abc' }, + }) .reply(200, '', { 'docker-content-digest': 'some-digest' }); mockEcrAuthResolve({ - authorizationData: [{ authorizationToken: 'abcdef' }], + authorizationData: [{ authorizationToken: 'abc' }], }); const res = await getDigest( @@ -281,20 +286,14 @@ describe('datasource/docker/index', () => { }, 'some-tag' ); - const trace = httpMock.getTrace(); + expect(res).toBe('some-digest'); - expect(trace[1].headers.authorization).toBe('Basic abcdef'); - expect(trace).toMatchSnapshot(); }); + it('continues without token if ECR authentication could not be extracted', async () => { - httpMock - .scope(amazonUrl) - .get('/') - .reply(200, '', { - 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', - }) - .get('/') - .reply(403); + httpMock.scope(amazonUrl).get('/').reply(401, '', { + 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', + }); mockEcrAuthResolve(); const res = await getDigest( @@ -305,18 +304,13 @@ describe('datasource/docker/index', () => { 'some-tag' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('continues without token if ECR authentication fails', async () => { hostRules.find.mockReturnValue({}); - httpMock - .scope(amazonUrl) - .get('/') - .reply(200, '', { - 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', - }) - .get('/') - .reply(403); + httpMock.scope(amazonUrl).get('/').reply(401, '', { + 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', + }); mockEcrAuthReject('some error'); const res = await getDigest( { @@ -326,8 +320,8 @@ describe('datasource/docker/index', () => { 'some-tag' ); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('continues without token, when no header is present', async () => { httpMock .scope(baseUrl) @@ -335,24 +329,24 @@ describe('datasource/docker/index', () => { .reply(200, '', { 'content-type': 'text/plain', }) - .get('/library/some-dep/manifests/some-new-value') + .head('/library/some-dep/manifests/some-new-value') .reply(200, {}, { 'docker-content-digest': 'some-digest' }); const res = await getDigest( { datasource: 'docker', depName: 'some-dep' }, 'some-new-value' ); expect(res).toBe('some-digest'); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('supports scoped names', async () => { httpMock .scope(baseUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:samalba/my-app:pull "', }) - .get('/library/some-other-dep/manifests/8.0.0-alpine') + .head('/library/some-other-dep/manifests/8.0.0-alpine') .reply(200, {}, { 'docker-content-digest': 'some-digest' }); httpMock .scope(authUrl) @@ -365,14 +359,15 @@ describe('datasource/docker/index', () => { '8.0.0-alpine' ); expect(res).toBe('some-digest'); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('should throw error for 429', async () => { httpMock.scope(baseUrl).get('/').replyWithError({ statusCode: 429 }); await expect( getDigest({ datasource: 'docker', depName: 'some-dep' }, 'latest') ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); + it('should throw error for 5xx', async () => { httpMock.scope(baseUrl).get('/').replyWithError({ statusCode: 504 }); await expect( @@ -380,6 +375,7 @@ describe('datasource/docker/index', () => { ).rejects.toThrow(EXTERNAL_HOST_ERROR); }); }); + describe('getReleases', () => { it('returns null if no token', async () => { httpMock @@ -394,8 +390,8 @@ describe('datasource/docker/index', () => { registryUrls: ['https://docker.io'], }); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('uses custom registry with registryUrls', async () => { const tags = ['1.0.0']; httpMock @@ -425,8 +421,8 @@ describe('datasource/docker/index', () => { }; const res = await getPkgReleases(config); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('uses custom registry in depName', async () => { const tags = ['1.0.0']; httpMock @@ -444,8 +440,8 @@ describe('datasource/docker/index', () => { depName: 'registry.company.com/node', }); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('uses quay api', async () => { const tags = [{ name: '5.0.12' }]; httpMock @@ -453,7 +449,11 @@ describe('datasource/docker/index', () => { .get( '/api/v1/repository/bitnami/redis/tag/?limit=100&page=1&onlyActiveTags=true' ) - .reply(200, { tags, has_additional: false }) + .reply(200, { tags, has_additional: true }) + .get( + '/api/v1/repository/bitnami/redis/tag/?limit=100&page=2&onlyActiveTags=true' + ) + .reply(200, { tags: [], has_additional: false }) .get('/v2/') .reply(200, '', {}) .get('/v2/bitnami/redis/manifests/5.0.12') @@ -465,8 +465,8 @@ describe('datasource/docker/index', () => { }; const res = await getPkgReleases(config); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('uses quay api and test error', async () => { httpMock .scope('https://quay.io') @@ -483,6 +483,7 @@ describe('datasource/docker/index', () => { 'external-host-error' ); }); + it('uses lower tag limit for ECR deps', async () => { httpMock .scope(amazonUrl) @@ -491,23 +492,28 @@ describe('datasource/docker/index', () => { // The tag limit parameter `n` needs to be limited to 1000 for ECR // See https://docs.aws.amazon.com/AmazonECR/latest/APIReference/API_DescribeRepositories.html#ECR-DescribeRepositories-request-maxResults .get('/node/tags/list?n=1000') - .reply(200, {}, {}) + .reply(200, { tags: ['some'] }, {}) .get('/') .reply(200, '', {}) - .get('/node/manifests/undefined') + .get('/node/manifests/some') .reply(200); - await getPkgReleases({ - datasource: id, - depName: '123456789.dkr.ecr.us-east-1.amazonaws.com/node', + expect( + await getPkgReleases({ + datasource: id, + depName: '123456789.dkr.ecr.us-east-1.amazonaws.com/node', + }) + ).toEqual({ + registryUrl: 'https://123456789.dkr.ecr.us-east-1.amazonaws.com', + releases: [], }); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('adds library/ prefix for Docker Hub (implicit)', async () => { const tags = ['1.0.0']; httpMock .scope(baseUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/node:pull "', }) @@ -528,14 +534,14 @@ describe('datasource/docker/index', () => { depName: 'node', }); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('adds library/ prefix for Docker Hub (explicit)', async () => { const tags = ['1.0.0']; httpMock .scope(baseUrl) .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:library/node:pull "', }) @@ -556,14 +562,14 @@ describe('datasource/docker/index', () => { depName: 'docker.io/node', }); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('adds no library/ prefix for other registries', async () => { const tags = ['1.0.0']; httpMock .scope('https://k8s.gcr.io/v2/') .get('/') - .reply(200, '', { + .reply(401, '', { 'www-authenticate': 'Bearer realm="https://k8s.gcr.io/v2/token",service="k8s.gcr.io"', }) @@ -582,8 +588,8 @@ describe('datasource/docker/index', () => { depName: 'k8s.gcr.io/kubernetes-dashboard-amd64', }); expect(res.releases).toHaveLength(1); - expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('returns null on error', async () => { httpMock .scope(baseUrl) @@ -596,25 +602,44 @@ describe('datasource/docker/index', () => { depName: 'my/node', }); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('returns null if no auth', async () => { - hostRules.find.mockReturnValue({}); + it('strips trailing slash from registry', async () => { httpMock .scope(baseUrl) .get('/') - .reply(200, undefined, { - 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', + .reply(401, '', { + 'www-authenticate': + 'Bearer realm="https://auth.docker.io/token",service="registry.docker.io",scope="repository:my/node:pull "', }) + .get('/my/node/tags/list?n=10000') + .reply(200, { tags: ['1.0.0'] }, {}) .get('/') - .reply(403); + .reply(200) + .get('/my/node/manifests/1.0.0') + .reply(200); + httpMock + .scope(authUrl) + .get('/token?service=registry.docker.io&scope=repository:my/node:pull') + .reply(200, { token: 'some-token ' }); + const res = await getPkgReleases({ + datasource: id, + depName: 'my/node', + registryUrls: ['https://index.docker.io/'], + }); + expect(res?.releases).toHaveLength(1); + }); + + it('returns null if no auth', async () => { + hostRules.find.mockReturnValue({}); + httpMock.scope(baseUrl).get('/').reply(401, undefined, { + 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', + }); const res = await getPkgReleases({ datasource: id, depName: 'node', }); expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); }); it('supports labels', async () => { @@ -654,9 +679,7 @@ describe('datasource/docker/index', () => { datasource: id, depName: 'registry.company.com/node', }); - const trace = httpMock.getTrace(); expect(res).toMatchSnapshot(); - expect(trace).toMatchSnapshot(); }); it('supports manifest lists', async () => { @@ -692,9 +715,7 @@ describe('datasource/docker/index', () => { datasource: id, depName: 'registry.company.com/node', }); - const trace = httpMock.getTrace(); expect(res).toMatchSnapshot(); - expect(trace).toMatchSnapshot(); }); it('ignores unsupported manifest', async () => { @@ -714,9 +735,7 @@ describe('datasource/docker/index', () => { datasource: id, depName: 'registry.company.com/node', }); - const trace = httpMock.getTrace(); expect(res).toMatchSnapshot(); - expect(trace).toMatchSnapshot(); }); it('ignores unsupported schema version', async () => { @@ -733,17 +752,25 @@ describe('datasource/docker/index', () => { datasource: id, depName: 'registry.company.com/node', }); - const trace = httpMock.getTrace(); expect(res).toMatchSnapshot(); - expect(trace).toMatchSnapshot(); }); it('supports redirect', async () => { httpMock - .scope('https://registry.company.com/v2') + .scope('https://registry.company.com/v2', { + badheaders: ['authorization'], + }) .get('/') .times(3) - .reply(200) + .reply(401, '', { + 'www-authenticate': 'Basic realm="My Private Docker Registry Server"', + }); + httpMock + .scope('https://registry.company.com/v2', { + reqheaders: { + authorization: 'Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk', + }, + }) .get('/node/tags/list?n=10000') .reply(200, { tags: ['latest'] }) .get('/node/manifests/latest') @@ -758,7 +785,7 @@ describe('datasource/docker/index', () => { 'https://abc.s3.amazon.com/some-config-digest?X-Amz-Algorithm=xxxx', }); httpMock - .scope('https://abc.s3.amazon.com') + .scope('https://abc.s3.amazon.com', { badheaders: ['authorization'] }) .get('/some-config-digest') .query({ 'X-Amz-Algorithm': 'xxxx' }) .reply(200, { @@ -768,13 +795,7 @@ describe('datasource/docker/index', () => { datasource: id, depName: 'registry.company.com/node', }); - const trace = httpMock.getTrace(); expect(res).toMatchSnapshot(); - expect(trace).toMatchSnapshot(); - expect(trace[1].headers.authorization).toBe( - 'Basic c29tZS11c2VybmFtZTpzb21lLXBhc3N3b3Jk' - ); - expect(trace[trace.length - 1].headers.authorization).toBeUndefined(); }); }); }); diff --git a/lib/datasource/docker/index.ts b/lib/datasource/docker/index.ts index 64a32e1505accf..c76659ef57091f 100644 --- a/lib/datasource/docker/index.ts +++ b/lib/datasource/docker/index.ts @@ -3,6 +3,7 @@ import parseLinkHeader from 'parse-link-header'; import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; +import { hasKey } from '../../util/object'; import { ensurePathPrefix } from '../../util/url'; import { api as dockerVersioning, @@ -12,7 +13,7 @@ import type { GetReleasesConfig, ReleaseResult } from '../types'; import { defaultRegistryUrls, ecrRegex, - extractDigestFromResponse, + extractDigestFromResponseBody, getAuthHeaders, getLabels, getManifestResponse, @@ -75,7 +76,10 @@ async function getDockerApiTags( } let page = 1; do { - const res = await http.getJson<{ tags: string[] }>(url, { headers }); + const res = await http.getJson<{ tags: string[] }>(url, { + headers, + noAuth: true, + }); tags = tags.concat(res.body.tags); const linkHeader = parseLinkHeader(res.headers.link as string); url = linkHeader?.next ? URL.resolve(url, linkHeader.next.url) : null; @@ -188,13 +192,27 @@ export async function getDigest( if (cachedResult !== undefined) { return cachedResult; } - const manifestResponse = await getManifestResponse( + let manifestResponse = await getManifestResponse( registryHost, dockerRepository, - newTag + newTag, + 'head' ); if (manifestResponse) { - digest = extractDigestFromResponse(manifestResponse) || null; + if (hasKey('docker-content-digest', manifestResponse.headers)) { + digest = manifestResponse.headers['docker-content-digest'] || null; + } else { + logger.debug( + { registryHost }, + 'Missing docker content digest header, pulling full manifest' + ); + manifestResponse = await getManifestResponse( + registryHost, + dockerRepository, + newTag + ); + digest = extractDigestFromResponseBody(manifestResponse); + } logger.debug({ digest }, 'Got docker digest'); } } catch (err) /* istanbul ignore next */ { @@ -240,6 +258,7 @@ export async function getReleases({ } const releases = tags.map((version) => ({ version })); const ret: ReleaseResult = { + registryUrl: registryHost, releases, }; diff --git a/lib/datasource/github-releases/test/index.ts b/lib/datasource/github-releases/test/index.ts index 7c40f0962f18f7..95c756a089d9eb 100644 --- a/lib/datasource/github-releases/test/index.ts +++ b/lib/datasource/github-releases/test/index.ts @@ -20,7 +20,7 @@ export class GitHubReleaseMocker { published_at: '2020-03-09T11:00:00Z', prerelease: false, assets: [], - }; + } as GithubRelease; for (const assetFn of Object.keys(assets)) { const assetPath = `/repos/${this.lookupName}/releases/download/${version}/${assetFn}`; const assetData = assets[assetFn]; diff --git a/lib/datasource/github-releases/types.ts b/lib/datasource/github-releases/types.ts index 78dd7af200b933..2989ddcaee72fc 100644 --- a/lib/datasource/github-releases/types.ts +++ b/lib/datasource/github-releases/types.ts @@ -1,8 +1,13 @@ export type GithubRelease = { + id: number; tag_name: string; published_at: string; prerelease: boolean; assets: GithubReleaseAsset[]; + + html_url: string; + name: string; + body: string; }; export interface GithubReleaseAsset { diff --git a/lib/datasource/github-tags/index.ts b/lib/datasource/github-tags/index.ts index 2300a58e296cf3..9099448e617d2d 100644 --- a/lib/datasource/github-tags/index.ts +++ b/lib/datasource/github-tags/index.ts @@ -4,7 +4,7 @@ import { GithubHttp } from '../../util/http/github'; import { ensureTrailingSlash } from '../../util/url'; import * as githubReleases from '../github-releases'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; -import type { TagResponse } from './types'; +import type { GitHubTag, TagResponse } from './types'; export const id = 'github-tags'; export const customRegistrySupport = true; @@ -148,12 +148,9 @@ async function getTags({ : `${sourceUrlBase}api/v3/`; // tag const url = `${apiBaseUrl}repos/${repo}/tags?per_page=100`; - type GitHubTag = { - name: string; - }[]; const versions = ( - await http.getJson(url, { + await http.getJson(url, { paginate: true, }) ).body.map((o) => o.name); diff --git a/lib/datasource/github-tags/types.ts b/lib/datasource/github-tags/types.ts index 0c570048a2fcc7..e8b5888166c0b2 100644 --- a/lib/datasource/github-tags/types.ts +++ b/lib/datasource/github-tags/types.ts @@ -5,3 +5,7 @@ export interface TagResponse { sha: string; }; } + +export interface GitHubTag { + name: string; +} diff --git a/lib/datasource/gitlab-releases/__snapshots__/index.spec.ts.snap b/lib/datasource/gitlab-releases/__snapshots__/index.spec.ts.snap new file mode 100644 index 00000000000000..f39b80f4c89cfb --- /dev/null +++ b/lib/datasource/gitlab-releases/__snapshots__/index.spec.ts.snap @@ -0,0 +1,43 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`datasource/gitlab-releases/index getReleases returns releases from custom registry 1`] = ` +Object { + "registryUrl": "https://gitlab.company.com", + "releases": Array [ + Object { + "gitRef": "v1.0.0", + "registryUrl": "https://gitlab.company.com", + "releaseTimestamp": "2021-01-01T00:00:00.000Z", + "version": "v1.0.0", + }, + Object { + "gitRef": "v1.1.0", + "registryUrl": "https://gitlab.company.com", + "releaseTimestamp": "2021-03-01T00:00:00.000Z", + "version": "v1.1.0", + }, + ], + "sourceUrl": "https://gitlab.company.com/some/dep2", +} +`; + +exports[`datasource/gitlab-releases/index getReleases returns releases from default registry 1`] = ` +Object { + "registryUrl": "https://gitlab.com", + "releases": Array [ + Object { + "gitRef": "v1.0.0", + "registryUrl": "https://gitlab.com", + "releaseTimestamp": "2021-01-01T00:00:00.000Z", + "version": "v1.0.0", + }, + Object { + "gitRef": "v1.1.0", + "registryUrl": "https://gitlab.com", + "releaseTimestamp": "2021-03-01T00:00:00.000Z", + "version": "v1.1.0", + }, + ], + "sourceUrl": "https://gitlab.com/some/dep2", +} +`; diff --git a/lib/datasource/gitlab-releases/index.spec.ts b/lib/datasource/gitlab-releases/index.spec.ts new file mode 100644 index 00000000000000..e4a438d007619a --- /dev/null +++ b/lib/datasource/gitlab-releases/index.spec.ts @@ -0,0 +1,58 @@ +import { getPkgReleases } from '..'; +import * as httpMock from '../../../test/http-mock'; +import { GitlabReleasesDatasource } from '.'; + +describe('datasource/gitlab-releases/index', () => { + describe('getReleases', () => { + const body = [ + { + tag_name: 'v1.0.0', + released_at: '2021-01-01T00:00:00.000Z', + }, + { + tag_name: 'v1.1.0', + released_at: '2021-03-01T00:00:00.000Z', + }, + ]; + + it('returns releases from custom registry', async () => { + httpMock + .scope('https://gitlab.company.com') + .get('/api/v4/projects/some%2Fdep2/releases') + .reply(200, body); + const res = await getPkgReleases({ + datasource: GitlabReleasesDatasource.id, + registryUrls: ['https://gitlab.company.com'], + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(2); + }); + + it('returns releases from default registry', async () => { + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/some%2Fdep2/releases') + .reply(200, body); + const res = await getPkgReleases({ + datasource: GitlabReleasesDatasource.id, + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res.releases).toHaveLength(2); + }); + + it('return null if not found', async () => { + httpMock + .scope('https://gitlab.com') + .get('/api/v4/projects/some%2Fdep2/releases') + .reply(404); + expect( + await getPkgReleases({ + datasource: GitlabReleasesDatasource.id, + depName: 'some/dep2', + }) + ).toBeNull(); + }); + }); +}); diff --git a/lib/datasource/gitlab-releases/index.ts b/lib/datasource/gitlab-releases/index.ts new file mode 100644 index 00000000000000..6a4cec10bf410c --- /dev/null +++ b/lib/datasource/gitlab-releases/index.ts @@ -0,0 +1,54 @@ +import { cache } from '../../util/cache/package/decorator'; +import { GitlabHttp } from '../../util/http/gitlab'; +import { Datasource } from '../datasource'; +import type { GetReleasesConfig, Release, ReleaseResult } from '../types'; +import type { GitlabRelease } from './types'; + +export class GitlabReleasesDatasource extends Datasource { + static readonly id = 'gitlab-releases'; + + override readonly defaultRegistryUrls = ['https://gitlab.com']; + + static readonly registryStrategy = 'first'; + + constructor() { + super(GitlabReleasesDatasource.id); + this.http = new GitlabHttp(GitlabReleasesDatasource.id); + } + + @cache({ + namespace: `datasource-${GitlabReleasesDatasource.id}`, + key: ({ registryUrl, lookupName }: GetReleasesConfig) => + `${registryUrl}/${lookupName}`, + }) + async getReleases({ + registryUrl, + lookupName, + }: GetReleasesConfig): Promise { + const urlEncodedRepo = encodeURIComponent(lookupName); + const apiUrl = `${registryUrl}/api/v4/projects/${urlEncodedRepo}/releases`; + + try { + const gitlabReleasesResponse = ( + await this.http.getJson(apiUrl) + ).body; + + return { + sourceUrl: `${registryUrl}/${lookupName}`, + releases: gitlabReleasesResponse.map(({ tag_name, released_at }) => { + const release: Release = { + registryUrl, + gitRef: tag_name, + version: tag_name, + releaseTimestamp: released_at, + }; + return release; + }), + }; + } catch (e) { + this.handleGenericErrors(e); + } + /* istanbul ignore next */ + return null; + } +} diff --git a/lib/datasource/gitlab-releases/types.ts b/lib/datasource/gitlab-releases/types.ts new file mode 100644 index 00000000000000..246ba0a9ed483f --- /dev/null +++ b/lib/datasource/gitlab-releases/types.ts @@ -0,0 +1,6 @@ +export interface GitlabRelease { + description: string; + name: string; + tag_name: string; + released_at: string; +} diff --git a/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap b/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap index 4ab77e4ae0e3e7..4ab20c12a01f6f 100644 --- a/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/gitlab-tags/__snapshots__/index.spec.ts.snap @@ -2,7 +2,7 @@ exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry 1`] = ` Object { - "registryUrl": "https://gitlab.company.com/api/v4/", + "registryUrl": "https://gitlab.company.com/api/v4", "releases": Array [ Object { "gitRef": "v1.0.0", @@ -18,23 +18,30 @@ Object { "version": "v1.1.1", }, ], - "sourceUrl": "https://gitlab.company.com/api/v4/some/dep2", + "sourceUrl": "https://gitlab.company.com/some/dep2", } `; -exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry 2`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "host": "gitlab.company.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", +exports[`datasource/gitlab-tags/index getReleases returns tags from custom registry in sub path 1`] = ` +Object { + "registryUrl": "https://my.company.com/gitlab", + "releases": Array [ + Object { + "gitRef": "v1.0.0", + "releaseTimestamp": "2020-03-04T18:01:37.000Z", + "version": "v1.0.0", + }, + Object { + "gitRef": "v1.1.0", + "version": "v1.1.0", }, - "method": "GET", - "url": "https://gitlab.company.com/api/v4/projects/some%2Fdep2/repository/tags?per_page=100", - }, -] + Object { + "gitRef": "v1.1.1", + "version": "v1.1.1", + }, + ], + "sourceUrl": "https://my.company.com/gitlab/some/dep2", +} `; exports[`datasource/gitlab-tags/index getReleases returns tags with default registry 1`] = ` @@ -53,18 +60,3 @@ Object { "sourceUrl": "https://gitlab.com/some/dep2", } `; - -exports[`datasource/gitlab-tags/index getReleases returns tags with default registry 2`] = ` -Array [ - Object { - "headers": Object { - "accept": "application/json", - "accept-encoding": "gzip, deflate, br", - "host": "gitlab.com", - "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", - }, - "method": "GET", - "url": "https://gitlab.com/api/v4/projects/some%2Fdep2/repository/tags?per_page=100", - }, -] -`; diff --git a/lib/datasource/gitlab-tags/index.spec.ts b/lib/datasource/gitlab-tags/index.spec.ts index 76ea7a59338b17..0f468cf8a7db6c 100644 --- a/lib/datasource/gitlab-tags/index.spec.ts +++ b/lib/datasource/gitlab-tags/index.spec.ts @@ -31,7 +31,35 @@ describe('datasource/gitlab-tags/index', () => { }); expect(res).toMatchSnapshot(); expect(res.releases).toHaveLength(3); - expect(httpMock.getTrace()).toMatchSnapshot(); + }); + + it('returns tags from custom registry in sub path', async () => { + const body = [ + { + name: 'v1.0.0', + commit: { + created_at: '2020-03-04T12:01:37.000-06:00', + }, + }, + { + name: 'v1.1.0', + commit: {}, + }, + { + name: 'v1.1.1', + }, + ]; + httpMock + .scope('https://my.company.com/gitlab') + .get('/api/v4/projects/some%2Fdep2/repository/tags?per_page=100') + .reply(200, body); + const res = await getPkgReleases({ + datasource, + registryUrls: ['https://my.company.com/gitlab'], + depName: 'some/dep2', + }); + expect(res).toMatchSnapshot(); + expect(res?.releases).toHaveLength(3); }); it('returns tags with default registry', async () => { @@ -46,7 +74,6 @@ describe('datasource/gitlab-tags/index', () => { }); expect(res).toMatchSnapshot(); expect(res.releases).toHaveLength(2); - expect(httpMock.getTrace()).toMatchSnapshot(); }); }); }); diff --git a/lib/datasource/gitlab-tags/index.ts b/lib/datasource/gitlab-tags/index.ts index 6c93462dc32ef4..fde3a5f86974b5 100644 --- a/lib/datasource/gitlab-tags/index.ts +++ b/lib/datasource/gitlab-tags/index.ts @@ -1,12 +1,12 @@ -import URL from 'url'; import * as packageCache from '../../util/cache/package'; import { GitlabHttp } from '../../util/http/gitlab'; +import { joinUrlParts } from '../../util/url'; import type { GetReleasesConfig, ReleaseResult } from '../types'; import type { GitlabTag } from './types'; -const gitlabApi = new GitlabHttp(); - export const id = 'gitlab-tags'; +const gitlabApi = new GitlabHttp(id); + export const customRegistrySupport = true; export const defaultRegistryUrls = ['https://gitlab.com']; export const registryStrategy = 'first'; @@ -19,9 +19,11 @@ function getCacheKey(depHost: string, repo: string): string { } export async function getReleases({ - registryUrl: depHost, + registryUrl, lookupName: repo, }: GetReleasesConfig): Promise { + const depHost = registryUrl.replace(/\/api\/v4$/, ''); + const cachedResult = await packageCache.get( cacheNamespace, getCacheKey(depHost, repo) @@ -34,9 +36,11 @@ export async function getReleases({ const urlEncodedRepo = encodeURIComponent(repo); // tag - const url = URL.resolve( + const url = joinUrlParts( depHost, - `/api/v4/projects/${urlEncodedRepo}/repository/tags?per_page=100` + `api/v4/projects`, + urlEncodedRepo, + `repository/tags?per_page=100` ); const gitlabTags = ( @@ -46,7 +50,7 @@ export async function getReleases({ ).body; const dependency: ReleaseResult = { - sourceUrl: URL.resolve(depHost, repo), + sourceUrl: joinUrlParts(depHost, repo), releases: null, }; dependency.releases = gitlabTags.map(({ name, commit }) => ({ diff --git a/lib/datasource/index.ts b/lib/datasource/index.ts index 1b12c8b5b8911a..1d914e800d9ac7 100644 --- a/lib/datasource/index.ts +++ b/lib/datasource/index.ts @@ -7,6 +7,7 @@ import * as memCache from '../util/cache/memory'; import * as packageCache from '../util/cache/package'; import { clone } from '../util/clone'; import { regEx } from '../util/regex'; +import { trimTrailingSlash } from '../util/url'; import * as allVersioning from '../versioning'; import datasources from './api'; import { addMetaData } from './metadata'; @@ -200,7 +201,7 @@ function resolveRegistryUrls( } else { registryUrls = [...defaultRegistryUrls]; } - return registryUrls.filter(Boolean); + return registryUrls.filter(Boolean).map(trimTrailingSlash); } export function getDefaultVersioning(datasourceName: string): string { diff --git a/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap b/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap index 19e59eebf09b64..0a310669ee9904 100644 --- a/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/jenkins-plugins/__snapshots__/index.spec.ts.snap @@ -9,7 +9,7 @@ Object { exports[`datasource/jenkins-plugins/index getReleases returns package releases for a hit for info and releases 1`] = ` Object { - "registryUrl": "https://updates.jenkins.io/", + "registryUrl": "https://updates.jenkins.io", "releases": Array [ Object { "downloadUrl": "http://updates.jenkins-ci.org/download/plugins/email-ext/2.10/email-ext.hpi", diff --git a/lib/datasource/nuget/__snapshots__/index.spec.ts.snap b/lib/datasource/nuget/__snapshots__/index.spec.ts.snap index 8e5f4830f0d026..2b976f37d4d941 100644 --- a/lib/datasource/nuget/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/nuget/__snapshots__/index.spec.ts.snap @@ -67,7 +67,7 @@ Array [ exports[`datasource/nuget/index getReleases handles paginated results (v2) 1`] = ` Object { - "registryUrl": "https://www.nuget.org/api/v2/", + "registryUrl": "https://www.nuget.org/api/v2", "releases": Array [ Object { "version": "1.0.0", @@ -104,7 +104,7 @@ Array [ exports[`datasource/nuget/index getReleases processes real data (v2) 1`] = ` Object { - "registryUrl": "https://www.nuget.org/api/v2/", + "registryUrl": "https://www.nuget.org/api/v2", "releases": Array [ Object { "releaseTimestamp": "2011-01-07T07:57:55.387Z", @@ -2052,7 +2052,7 @@ Array [ exports[`datasource/nuget/index getReleases processes real data with no github project url (v2) 1`] = ` Object { - "registryUrl": "https://www.nuget.org/api/v2/", + "registryUrl": "https://www.nuget.org/api/v2", "releases": Array [ Object { "version": "3.11.0", @@ -2078,7 +2078,7 @@ Array [ exports[`datasource/nuget/index getReleases processes real data without project url (v2) 1`] = ` Object { - "registryUrl": "https://www.nuget.org/api/v2/", + "registryUrl": "https://www.nuget.org/api/v2", "releases": Array [ Object { "version": "2.5.7.10213", diff --git a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap index 623cd7804a3895..a3bccdc7c17b80 100644 --- a/lib/datasource/pypi/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/pypi/__snapshots__/index.spec.ts.snap @@ -83,7 +83,7 @@ Array [ exports[`datasource/pypi/index getReleases parses data-requires-python and respects constraints from simple endpoint 1`] = ` Object { - "registryUrl": "https://pypi.org/simple/", + "registryUrl": "https://pypi.org/simple", "releases": Array [ Object { "version": "0.1.2", @@ -123,7 +123,7 @@ Array [ exports[`datasource/pypi/index getReleases process data from +simple endpoint 1`] = ` Object { - "registryUrl": "https://some.registry.org/+simple/", + "registryUrl": "https://some.registry.org/+simple", "releases": Array [ Object { "version": "0.1.2", @@ -179,7 +179,7 @@ Array [ exports[`datasource/pypi/index getReleases process data from simple endpoint 1`] = ` Object { - "registryUrl": "https://pypi.org/simple/", + "registryUrl": "https://pypi.org/simple", "releases": Array [ Object { "version": "0.1.2", @@ -235,7 +235,7 @@ Array [ exports[`datasource/pypi/index getReleases process data from simple endpoint with hyphens replaced with underscores 1`] = ` Object { - "registryUrl": "https://pypi.org/simple/", + "registryUrl": "https://pypi.org/simple", "releases": Array [ Object { "version": "0.0.5", @@ -260,7 +260,7 @@ Array [ exports[`datasource/pypi/index getReleases processes real data 1`] = ` Object { - "registryUrl": "https://pypi.org/pypi/", + "registryUrl": "https://pypi.org/pypi", "releases": Array [ Object { "releaseTimestamp": "2017-04-03T16:55:14.000Z", @@ -373,7 +373,7 @@ Array [ exports[`datasource/pypi/index getReleases respects constraints 1`] = ` Object { - "registryUrl": "https://pypi.org/pypi/", + "registryUrl": "https://pypi.org/pypi", "releases": Array [ Object { "version": "0.4.0", diff --git a/lib/datasource/repology/__snapshots__/index.spec.ts.snap b/lib/datasource/repology/__snapshots__/index.spec.ts.snap index a21d22412b39bb..f31c48d6160b24 100644 --- a/lib/datasource/repology/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/repology/__snapshots__/index.spec.ts.snap @@ -2,7 +2,7 @@ exports[`datasource/repology/index getReleases returns correct version for api package 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "1.181", @@ -38,7 +38,7 @@ Array [ exports[`datasource/repology/index getReleases returns correct version for binary package 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "1.14.2-2+deb10u1", @@ -64,7 +64,7 @@ Array [ exports[`datasource/repology/index getReleases returns correct version for multi-package project with different name 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "12.2-4+deb10u1", @@ -90,7 +90,7 @@ Array [ exports[`datasource/repology/index getReleases returns correct version for multi-package project with same name 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "9.3.0-r2", @@ -116,7 +116,7 @@ Array [ exports[`datasource/repology/index getReleases returns correct version for source package 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "1.181", @@ -152,7 +152,7 @@ Array [ exports[`datasource/repology/index getReleases returns multiple versions if they are present in repository 1`] = ` Object { - "registryUrl": "https://repology.org/", + "registryUrl": "https://repology.org", "releases": Array [ Object { "version": "1:11.0.7.10-1.el8_1", diff --git a/lib/datasource/repology/index.ts b/lib/datasource/repology/index.ts index fd571da1018b8c..53fb34668248cd 100644 --- a/lib/datasource/repology/index.ts +++ b/lib/datasource/repology/index.ts @@ -3,7 +3,7 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; -import { getQueryString } from '../../util/url'; +import { getQueryString, joinUrlParts } from '../../util/url'; import type { GetReleasesConfig, ReleaseResult } from '../types'; import type { RepologyPackage, RepologyPackageType } from './types'; @@ -52,7 +52,7 @@ async function queryPackagesViaResolver( // Retrieve list of packages by looking up Repology project const packages = await queryPackages( - `${registryUrl}tools/project-by?${query}` + joinUrlParts(registryUrl, `tools/project-by?${query}`) ); return packages; @@ -65,7 +65,7 @@ async function queryPackagesViaAPI( // Directly query the package via the API. This will only work if `packageName` has the // same name as the repology project const packages = await queryPackages( - `${registryUrl}api/v1/project/${packageName}` + joinUrlParts(registryUrl, `api/v1/project`, packageName) ); return packages; @@ -170,7 +170,7 @@ async function getCachedPackage( pkgName: string ): Promise { // Fetch previous result from cache if available - const cacheKey = `${registryUrl}${repoName}/${pkgName}`; + const cacheKey = joinUrlParts(registryUrl, repoName, pkgName); const cachedResult = await packageCache.get( cacheNamespace, cacheKey diff --git a/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap b/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap index 2f12ed4e3c296b..11742fb1031a89 100644 --- a/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/rubygems/__snapshots__/index.spec.ts.snap @@ -1461,7 +1461,7 @@ Array [ exports[`datasource/rubygems/index getReleases uses multiple source urls 1`] = ` Object { "homepage": "http://rubyonrails.org", - "registryUrl": "https://firstparty.com/basepath/", + "registryUrl": "https://firstparty.com/basepath", "releases": Array [ Object { "releaseTimestamp": "2009-07-25T18:01:56.000Z", diff --git a/lib/logger/__snapshots__/err-serializer.spec.ts.snap b/lib/logger/__snapshots__/err-serializer.spec.ts.snap index d23c5513d6a929..6b2de4afa35c95 100644 --- a/lib/logger/__snapshots__/err-serializer.spec.ts.snap +++ b/lib/logger/__snapshots__/err-serializer.spec.ts.snap @@ -60,6 +60,7 @@ Object { "accept-encoding": "gzip, deflate, br", "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", }, + "hostType": "any", "http2": false, "method": "POST", "password": "***********", diff --git a/lib/logger/utils.ts b/lib/logger/utils.ts index 36945b4eed12e6..34e7085c10cb1f 100644 --- a/lib/logger/utils.ts +++ b/lib/logger/utils.ts @@ -66,6 +66,7 @@ export default function prepareError(err: Error): Record { const options: Record = { headers: clone(err.options.headers), url: err.options.url?.toString(), + hostType: err.options.context.hostType, }; response.options = options; diff --git a/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap b/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap index 41e944805da20c..0abb636aec22c8 100644 --- a/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/dockerfile/__snapshots__/extract.spec.ts.snap @@ -346,6 +346,22 @@ Array [ ] `; +exports[`manager/dockerfile/extract extractPackageFile() handles prefixes 1`] = ` +Array [ + Object { + "autoReplaceStringTemplate": "{{lookupName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": "18.04", + "datasource": "docker", + "depName": "ubuntu", + "depType": "final", + "lookupName": "amd64/ubuntu", + "replaceString": "amd64/ubuntu:18.04", + "versioning": "ubuntu", + }, +] +`; + exports[`manager/dockerfile/extract extractPackageFile() handles quay hosts with port 1`] = ` Object { "autoReplaceStringTemplate": "{{lookupName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", diff --git a/lib/manager/dockerfile/extract.spec.ts b/lib/manager/dockerfile/extract.spec.ts index 896745048dfb3b..1e64e85dd7fc8a 100644 --- a/lib/manager/dockerfile/extract.spec.ts +++ b/lib/manager/dockerfile/extract.spec.ts @@ -265,6 +265,20 @@ describe('manager/dockerfile/extract', () => { }, ]); }); + + it('handles prefixes', () => { + const res = extractPackageFile('FROM amd64/ubuntu:18.04\n').deps; + expect(res).toMatchSnapshot([ + { + currentValue: '18.04', + depName: 'ubuntu', + lookupName: 'amd64/ubuntu', + versioning: 'ubuntu', + autoReplaceStringTemplate: + '{{lookupName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}', + }, + ]); + }); }); describe('getDep()', () => { it('rejects null', () => { diff --git a/lib/manager/dockerfile/extract.ts b/lib/manager/dockerfile/extract.ts index a21cf10f9d16a8..4a0b6f11ec3f75 100644 --- a/lib/manager/dockerfile/extract.ts +++ b/lib/manager/dockerfile/extract.ts @@ -58,6 +58,10 @@ export function getDep( if (dep.depName.startsWith(`${prefix}/`)) { dep.lookupName = dep.depName; dep.depName = dep.depName.replace(`${prefix}/`, ''); + if (specifyReplaceString) { + dep.autoReplaceStringTemplate = + '{{lookupName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}'; + } } } } diff --git a/lib/manager/droneci/__snapshots__/extract.spec.ts.snap b/lib/manager/droneci/__snapshots__/extract.spec.ts.snap index 0ed8a05d30d5d5..a7fe8577091daa 100644 --- a/lib/manager/droneci/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/droneci/__snapshots__/extract.spec.ts.snap @@ -12,7 +12,7 @@ Array [ "replaceString": "elixir:1.8.1-alpine", }, Object { - "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "autoReplaceStringTemplate": "{{lookupName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", "currentDigest": "sha256:36adc17e9cceab32179d3314da9cb9c737ffb11f0de4e688f407ad6d9ca32201", "currentValue": "10.0.0", "datasource": "docker", diff --git a/lib/manager/npm/extract/__snapshots__/index.spec.ts.snap b/lib/manager/npm/extract/__snapshots__/index.spec.ts.snap index 30f6f6e5bac7e4..5037b09dd13ae6 100644 --- a/lib/manager/npm/extract/__snapshots__/index.spec.ts.snap +++ b/lib/manager/npm/extract/__snapshots__/index.spec.ts.snap @@ -15,6 +15,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -137,6 +138,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -299,6 +301,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -347,6 +350,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": "package-lock.json", "npmrc": undefined, @@ -413,6 +417,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -466,6 +471,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -601,6 +607,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": "lerna.json", + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -736,6 +743,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": "lerna.json", + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -871,6 +879,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -892,6 +901,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": "lerna.json", + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1029,6 +1039,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": "lerna.json", + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1050,6 +1061,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": "lerna.json", + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1073,6 +1085,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1210,6 +1223,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1327,6 +1341,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, @@ -1462,6 +1477,7 @@ Object { "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": true, }, "npmLock": undefined, "npmrc": undefined, diff --git a/lib/manager/npm/extract/index.ts b/lib/manager/npm/extract/index.ts index ebeb46e3c0e091..7b2156cfce42f3 100644 --- a/lib/manager/npm/extract/index.ts +++ b/lib/manager/npm/extract/index.ts @@ -369,6 +369,7 @@ export async function extractPackageFile( ...lockFiles, managerData: { lernaJsonFile, + yarnZeroInstall, }, lernaClient, lernaPackages, diff --git a/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap b/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap index 2ddd158992983b..36e86da88b9190 100644 --- a/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap +++ b/lib/manager/npm/post-update/__snapshots__/yarn.spec.ts.snap @@ -38,6 +38,33 @@ Object { } `; +exports[`manager/npm/post-update/yarn does not use global cache if zero install is detected 1`] = ` +Array [ + Object { + "cmd": "yarn install", + "options": Object { + "cwd": "some-dir", + "encoding": "utf-8", + "env": Object { + "CI": "true", + "HOME": "/home/user", + "HTTPS_PROXY": "https://example.com", + "HTTP_PROXY": "http://example.com", + "LANG": "en_US.UTF-8", + "LC_ALL": "en_US", + "NO_PROXY": "localhost", + "PATH": "/tmp/path", + "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", + "YARN_ENABLE_SCRIPTS": "0", + "YARN_HTTP_TIMEOUT": "100000", + }, + "maxBuffer": 10485760, + "timeout": 900000, + }, + }, +] +`; + exports[`manager/npm/post-update/yarn generates lock files using yarn v1.22.0 1`] = ` Array [ Object { @@ -159,6 +186,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_GLOBAL_FOLDER": "/tmp/renovate/cache/berry", @@ -187,6 +215,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_GLOBAL_FOLDER": "/tmp/renovate/cache/berry", @@ -210,6 +239,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_GLOBAL_FOLDER": "/tmp/renovate/cache/berry", @@ -238,6 +268,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_GLOBAL_FOLDER": "/tmp/renovate/cache/berry", "YARN_HTTP_TIMEOUT": "100000", @@ -260,6 +291,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_GLOBAL_FOLDER": "/tmp/renovate/cache/berry", "YARN_HTTP_TIMEOUT": "100000", @@ -287,6 +319,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_HTTP_TIMEOUT": "100000", }, @@ -308,6 +341,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_HTTP_TIMEOUT": "100000", }, @@ -434,6 +468,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_HTTP_TIMEOUT": "100000", @@ -461,6 +496,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_HTTP_TIMEOUT": "100000", @@ -483,6 +519,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_HTTP_TIMEOUT": "100000", @@ -596,6 +633,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_HTTP_TIMEOUT": "100000", @@ -618,6 +656,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_ENABLE_SCRIPTS": "0", "YARN_HTTP_TIMEOUT": "100000", @@ -645,6 +684,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_HTTP_TIMEOUT": "100000", }, @@ -666,6 +706,7 @@ Array [ "LC_ALL": "en_US", "NO_PROXY": "localhost", "PATH": "/tmp/path", + "YARN_ENABLE_GLOBAL_CACHE": "1", "YARN_ENABLE_IMMUTABLE_INSTALLS": "false", "YARN_HTTP_TIMEOUT": "100000", }, diff --git a/lib/manager/npm/post-update/yarn.spec.ts b/lib/manager/npm/post-update/yarn.spec.ts index 9b950ada230be6..b1a54229f28a03 100644 --- a/lib/manager/npm/post-update/yarn.spec.ts +++ b/lib/manager/npm/post-update/yarn.spec.ts @@ -81,10 +81,7 @@ describe('manager/npm/post-update/yarn', () => { stdout: '3.0.0', stderr: '', }); - fs.readFile.mockImplementation( - (filename, encoding) => - new Promise((resolve) => resolve('package-lock-contents')) - ); + fs.readFile.mockResolvedValueOnce('package-lock-contents'); const config = { constraints: { yarn: '3.0.0', @@ -97,6 +94,24 @@ describe('manager/npm/post-update/yarn', () => { expect(fixSnapshots(execSnapshots)).toMatchSnapshot(); }); + it('does not use global cache if zero install is detected', async () => { + const execSnapshots = mockExecAll(exec, { + stdout: '2.1.0', + stderr: '', + }); + fs.readFile.mockResolvedValueOnce('package-lock-contents'); + const config = { + constraints: { + yarn: '>= 2.0.0', + }, + postUpdateOptions: ['yarnDedupeFewer', 'yarnDedupeHighest'], + managerData: { yarnZeroInstall: true }, + }; + const res = await yarnHelper.generateLockFile('some-dir', {}, config); + expect(res.lockFile).toEqual('package-lock-contents'); + expect(fixSnapshots(execSnapshots)).toMatchSnapshot(); + }); + it.each([ ['1.22.0', '^1.10.0'], ['2.1.0', '>= 2.0.0'], diff --git a/lib/manager/npm/post-update/yarn.ts b/lib/manager/npm/post-update/yarn.ts index 5166f8ce47ffcc..24e15f9ef630bc 100644 --- a/lib/manager/npm/post-update/yarn.ts +++ b/lib/manager/npm/post-update/yarn.ts @@ -105,6 +105,10 @@ export async function generateLockFile( extraEnv.YARN_ENABLE_IMMUTABLE_INSTALLS = 'false'; extraEnv.YARN_HTTP_TIMEOUT = '100000'; extraEnv.YARN_GLOBAL_FOLDER = env.YARN_GLOBAL_FOLDER; + if (!config.managerData?.yarnZeroInstall) { + logger.debug('Enabling global cache as zero-install is not detected'); + extraEnv.YARN_ENABLE_GLOBAL_CACHE = '1'; + } } if (!getGlobalConfig().allowScripts || config.ignoreScripts) { if (isYarn1) { diff --git a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap index f6af2a1e5c7ddc..182047b7165274 100644 --- a/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap +++ b/lib/manager/terraform/lockfile/__snapshots__/index.spec.ts.snap @@ -202,7 +202,7 @@ Array [ ] `; -exports[`manager/terraform/lockfile/index update single dependency with exact constraint 1`] = ` +exports[`manager/terraform/lockfile/index update single dependency with exact constraint and and depType required_provider 1`] = ` Object { "contents": "# This file is maintained automatically by \\"terraform init\\". # Manual edits may be lost in future updates. @@ -259,7 +259,74 @@ provider \\"registry.terraform.io/hashicorp/random\\" { } `; -exports[`manager/terraform/lockfile/index update single dependency with exact constraint 2`] = ` +exports[`manager/terraform/lockfile/index update single dependency with exact constraint and and depType required_provider 2`] = ` +Array [ + Array [ + "https://registry.terraform.io", + "hashicorp/aws", + "3.36.0", + ], +] +`; + +exports[`manager/terraform/lockfile/index update single dependency with exact constraint and depType provider 1`] = ` +Object { + "contents": "# This file is maintained automatically by \\"terraform init\\". +# Manual edits may be lost in future updates. + +provider \\"registry.terraform.io/hashicorp/aws\\" { + version = \\"3.36.0\\" + constraints = \\"3.36.0\\" + hashes = [ + \\"h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=\\", + \\"h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=\\", + ] +} + +provider \\"registry.terraform.io/hashicorp/azurerm\\" { + version = \\"2.50.0\\" + constraints = \\"~> 2.50\\" + hashes = [ + \\"h1:Vr6WUm88s9hXGkyVjHtHsP2Jmc2ypQXn6ww7dXtvk1M=\\", + \\"zh:0c0688d5a743248f8646d39eb3645a4ac19fd7523ba1b47072fa3fb03b92b1b0\\", + \\"zh:2beb3a55ee970f87a9292ae96d57134be8a03d0566117e7be0fe0d9c1267e4ea\\", + \\"zh:38091b463fbafe5756420ce34c87845c2a391fec0cded27bdcbbca28febad382\\", + \\"zh:4ba455da3b37ba8f8b03ff2781121d9c54d0bd8afd76dfe67593011c475dd73f\\", + \\"zh:5d32b9ed871b3c3b774dc69f1fe14cdf7c1fd63d12bb5f21aad4bfbf75e5ee3d\\", + \\"zh:6c80cf90a3fc1e17d9caf67cc558c2ff91f8b25e29fdf00942f67711895be5c0\\", + \\"zh:c0a53e3165407999d10de7aaa983485d42797433c60b5775791ae299121279ed\\", + \\"zh:dab51d6d76041505aeebf20111febe8616ec465ca31dfb7901f5f5c23a5af095\\", + \\"zh:e1ad6399f6a6d799002206ee4cb7b794dbb2533b8c3c14502a4419955ec96bff\\", + \\"zh:e98f1d178d1e111b3f3449e27d305ce263071226fad3d86272e1bd161c26fd43\\", + \\"zh:eb76ec000c9c49a0bf730370c8880f671597bc01f7b7401ab301df7124c049ec\\", + ] +} + +provider \\"registry.terraform.io/hashicorp/random\\" { + version = \\"2.2.1\\" + constraints = \\"~> 2.2\\" + hashes = [ + \\"h1:Zg1Bpi6vr7b0H6no8kVDfEucn5pvNALivdrVKVHarGs=\\", + \\"zh:072ce92b0138ee65df2e4e2e6e5f6632fa12a7e6453b91399bad89291855d426\\", + \\"zh:5731987fe61051515f449033e456ee55207caf17ef41096eb82247810585f53b\\", + \\"zh:6f18b10175708bb5839e1f2082dcc02651b876786cd54ec415a091f3821807c3\\", + \\"zh:7fa7737661380d18cba3cdc71c4ec6f2fd281b9d61112f6b48d06ca8bbf97771\\", + \\"zh:8466cb8fbb4de887b23039082a6e3dc85aeabce86dd808e2a7a65e4e1c51dbae\\", + \\"zh:888c63417701c13bbe785ab11dc690d4803e6a2156318cf188970b7b6400b99e\\", + \\"zh:a231df55d36fbad1a6705f5d3be4f7459a73ec76117d13f22aa83c10fc610278\\", + \\"zh:b62d9a4cd64a2d229070260f4abfef476ebbd7c5511b43e9cdccf23ce938f630\\", + \\"zh:b6bd1a325f909bb93f7c9bef00eb306bef1e406cbdf557901d755a3e7a4a5448\\", + \\"zh:b9f59afc23cc5567075f76313214baa1e5ce909325229e23c9a4666f7b26e7f7\\", + \\"zh:d040220c09b8d9d6bd937572bd5b14bc069af2b883185a873460530d8a1de6e6\\", + \\"zh:f254c1f943eb016ae07ebe91b23f813dc79f2064616c65f98c8f64ce23be90c4\\", + ] +} +", + "name": ".terraform.lock.hcl", +} +`; + +exports[`manager/terraform/lockfile/index update single dependency with exact constraint and depType provider 2`] = ` Array [ Array [ "https://registry.terraform.io", diff --git a/lib/manager/terraform/lockfile/index.spec.ts b/lib/manager/terraform/lockfile/index.spec.ts index 091fe80984a7f2..731d287b5aef07 100644 --- a/lib/manager/terraform/lockfile/index.spec.ts +++ b/lib/manager/terraform/lockfile/index.spec.ts @@ -61,7 +61,7 @@ describe('manager/terraform/lockfile/index', () => { ).toBeNull(); }); - it('update single dependency with exact constraint', async () => { + it('update single dependency with exact constraint and depType provider', async () => { fs.readLocalFile.mockResolvedValueOnce(validLockfile as any); fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl'); @@ -79,7 +79,50 @@ describe('manager/terraform/lockfile/index', () => { const result = await updateArtifacts({ packageFileName: 'main.tf', - updatedDeps: [{ depName: 'hashicorp/aws', lookupName: 'hashicorp/aws' }], + updatedDeps: [ + { + depName: 'hashicorp/aws', + lookupName: 'hashicorp/aws', + depType: 'provider', + }, + ], + newPackageFileContent: '', + config: localConfig, + }); + expect(result).not.toBeNull(); + expect(result).toBeArrayOfSize(1); + expect(result[0].file).not.toBeNull(); + expect(result[0].file).toMatchSnapshot(); + + expect(mockHash.mock.calls).toBeArrayOfSize(1); + expect(mockHash.mock.calls).toMatchSnapshot(); + }); + + it('update single dependency with exact constraint and and depType required_provider', async () => { + fs.readLocalFile.mockResolvedValueOnce(validLockfile as any); + fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl'); + + mockHash.mockResolvedValueOnce([ + 'h1:lDsKRxDRXPEzA4AxkK4t+lJd3IQIP2UoaplJGjQSp2s=', + 'h1:6zB2hX7YIOW26OrKsLJn0uLMnjqbPNxcz9RhlWEuuSY=', + ]); + + const localConfig: UpdateArtifactsConfig = { + updateType: 'minor', + newVersion: '3.36.0', + newValue: '3.36.0', + ...config, + }; + + const result = await updateArtifacts({ + packageFileName: 'main.tf', + updatedDeps: [ + { + depName: 'hashicorp/aws', + lookupName: 'hashicorp/aws', + depType: 'required_provider', + }, + ], newPackageFileContent: '', config: localConfig, }); @@ -92,6 +135,29 @@ describe('manager/terraform/lockfile/index', () => { expect(mockHash.mock.calls).toMatchSnapshot(); }); + it('do not update dependency with depType module', async () => { + const localConfig: UpdateArtifactsConfig = { + updateType: 'minor', + newVersion: '3.36.0', + newValue: '3.36.0', + ...config, + }; + + const result = await updateArtifacts({ + packageFileName: 'main.tf', + updatedDeps: [ + { + depName: 'terraform-aws-modules/vpc/aws', + lookupName: 'terraform-aws-modules/vpc/aws', + depType: 'module', + }, + ], + newPackageFileContent: '', + config: localConfig, + }); + expect(result).toBeNull(); + }); + it('update single dependency with range constraint and minor update from private registry', async () => { fs.readLocalFile.mockResolvedValueOnce(validLockfile as any); fs.getSiblingFileName.mockReturnValueOnce('.terraform.lock.hcl'); @@ -113,6 +179,7 @@ describe('manager/terraform/lockfile/index', () => { updatedDeps: [ { depName: 'azurerm', + depType: 'provider', lookupName: 'azurerm', registryUrls: ['https://registry.example.com'], }, @@ -147,7 +214,13 @@ describe('manager/terraform/lockfile/index', () => { const result = await updateArtifacts({ packageFileName: 'main.tf', - updatedDeps: [{ depName: 'random', lookupName: 'hashicorp/random' }], + updatedDeps: [ + { + depName: 'random', + lookupName: 'hashicorp/random', + depType: 'provider', + }, + ], newPackageFileContent: '', config: localConfig, }); @@ -178,7 +251,13 @@ describe('manager/terraform/lockfile/index', () => { const result = await updateArtifacts({ packageFileName: 'test/main.tf', - updatedDeps: [{ depName: 'random', lookupName: 'hashicorp/random' }], + updatedDeps: [ + { + depName: 'random', + lookupName: 'hashicorp/random', + depType: 'provider', + }, + ], newPackageFileContent: '', config: localConfig, }); diff --git a/lib/manager/terraform/lockfile/index.ts b/lib/manager/terraform/lockfile/index.ts index 93ce5384291cf0..1171d4d4dd1c37 100644 --- a/lib/manager/terraform/lockfile/index.ts +++ b/lib/manager/terraform/lockfile/index.ts @@ -80,7 +80,9 @@ export async function updateArtifacts({ // update all locks in the file during maintenance --> only update version in constraints const maintenanceUpdates = await updateAllLocks(locks); updates.push(...maintenanceUpdates); - } else { + } else if ( + ['provider', 'required_provider'].includes(updatedDeps[0].depType) + ) { // update only specific locks but with constrain updates const dep = updatedDeps[0]; diff --git a/lib/manager/types.ts b/lib/manager/types.ts index 6f09f08baac548..fb2701cd6c0711 100644 --- a/lib/manager/types.ts +++ b/lib/manager/types.ts @@ -255,7 +255,9 @@ export interface ManagerApi { } // TODO: name and properties used by npm manager -export interface PostUpdateConfig extends Record { +export interface PostUpdateConfig> + extends Record, + ManagerData { updatedPackageFiles?: File[]; postUpdateOptions?: string[]; skipInstalls?: boolean; diff --git a/lib/types/platform/gitlab/index.spec.ts b/lib/types/platform/gitlab/index.spec.ts new file mode 100644 index 00000000000000..945f2e4476ccd0 --- /dev/null +++ b/lib/types/platform/gitlab/index.spec.ts @@ -0,0 +1,25 @@ +import { + PLATFORM_TYPE_GITHUB, + PLATFORM_TYPE_GITLAB, +} from '../../../constants/platforms'; +import { GitlabReleasesDatasource } from '../../../datasource/gitlab-releases'; +import { id as GL_TAGS_DS } from '../../../datasource/gitlab-tags'; +import { GITLAB_API_USING_HOST_TYPES } from './index'; + +describe('types/platform/gitlab/index', () => { + it('should be part of the GITLAB_API_USING_HOST_TYPES', () => { + expect(GITLAB_API_USING_HOST_TYPES.includes(GL_TAGS_DS)).toBeTrue(); + expect( + GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id) + ).toBeTrue(); + expect( + GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB) + ).toBeTrue(); + }); + + it('should be not part of the GITLAB_API_USING_HOST_TYPES ', () => { + expect( + GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITHUB) + ).toBeFalse(); + }); +}); diff --git a/lib/types/platform/gitlab/index.ts b/lib/types/platform/gitlab/index.ts index a67df13e1ae81c..999f9149b18189 100644 --- a/lib/types/platform/gitlab/index.ts +++ b/lib/types/platform/gitlab/index.ts @@ -1,3 +1,4 @@ +import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms'; import { GitTreeNode } from '../../git'; export type GitLabBranch = { @@ -12,3 +13,9 @@ export type GitlabTreeNode = { id: string; name: string; } & GitTreeNode; + +export const GITLAB_API_USING_HOST_TYPES = [ + PLATFORM_TYPE_GITLAB, + 'gitlab-releases', + 'gitlab-tags', +]; diff --git a/lib/types/www-authenticate.d.ts b/lib/types/www-authenticate.d.ts new file mode 100644 index 00000000000000..94e78b2c3bea9d --- /dev/null +++ b/lib/types/www-authenticate.d.ts @@ -0,0 +1,18 @@ +declare module 'www-authenticate' { + interface wwwAuthenticate { + (username: string, password: string): unknown; + } + + namespace wwwAuthenticate { + namespace parsers { + class WWW_Authenticate { + readonly scheme: string; + readonly parms: Record; + + constructor(value: string); + } + } + } + + export = wwwAuthenticate; +} diff --git a/lib/util/http/__snapshots__/gitlab.spec.ts.snap b/lib/util/http/__snapshots__/gitlab.spec.ts.snap index 8b1f605d78788a..56e7c378c343d3 100644 --- a/lib/util/http/__snapshots__/gitlab.spec.ts.snap +++ b/lib/util/http/__snapshots__/gitlab.spec.ts.snap @@ -107,3 +107,18 @@ Array [ }, ] `; + +exports[`util/http/gitlab supports different datasources 1`] = ` +Array [ + Object { + "headers": Object { + "accept-encoding": "gzip, deflate, br", + "authorization": "Bearer def", + "host": "gitlab.com", + "user-agent": "RenovateBot/0.0.0-semantic-release (https://github.com/renovatebot/renovate)", + }, + "method": "GET", + "url": "https://gitlab.com/api/v4/some-url", + }, +] +`; diff --git a/lib/util/http/auth.ts b/lib/util/http/auth.ts index 4a7671eeb85900..b3a5383dfc5c42 100644 --- a/lib/util/http/auth.ts +++ b/lib/util/http/auth.ts @@ -1,17 +1,17 @@ import is from '@sindresorhus/is'; import { NormalizedOptions } from 'got'; -import { - PLATFORM_TYPE_GITEA, - PLATFORM_TYPE_GITLAB, -} from '../../constants/platforms'; +import { PLATFORM_TYPE_GITEA } from '../../constants/platforms'; import { GITHUB_API_USING_HOST_TYPES } from '../../types'; +import { GITLAB_API_USING_HOST_TYPES } from '../../types/platform/gitlab'; import { GotOptions } from './types'; export function applyAuthorization(inOptions: GotOptions): GotOptions { - const options = { ...inOptions }; - if (options.headers?.authorization) { + const options: GotOptions = { ...inOptions }; + + if (options.headers?.authorization || options.noAuth) { return options; } + if (options.token) { if (options.hostType === PLATFORM_TYPE_GITEA) { options.headers.authorization = `token ${options.token}`; @@ -27,7 +27,7 @@ export function applyAuthorization(inOptions: GotOptions): GotOptions { ); } } - } else if (options.hostType === PLATFORM_TYPE_GITLAB) { + } else if (GITLAB_API_USING_HOST_TYPES.includes(options.hostType)) { // GitLab versions earlier than 12.2 only support authentication with // a personal access token, which is 20 characters long. if (options.token.length === 20) { diff --git a/lib/util/http/gitlab.spec.ts b/lib/util/http/gitlab.spec.ts index acd7c6d9892eee..13f6a3962f6c35 100644 --- a/lib/util/http/gitlab.spec.ts +++ b/lib/util/http/gitlab.spec.ts @@ -1,14 +1,10 @@ import * as httpMock from '../../../test/http-mock'; import { EXTERNAL_HOST_ERROR } from '../../constants/error-messages'; import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; +import { GitlabReleasesDatasource } from '../../datasource/gitlab-releases'; import * as hostRules from '../host-rules'; import { GitlabHttp, setBaseUrl } from './gitlab'; -hostRules.add({ - hostType: PLATFORM_TYPE_GITLAB, - token: 'abc123', -}); - const gitlabApiHost = 'https://gitlab.com'; const selfHostedUrl = 'http://mycompany.com/gitlab'; @@ -19,10 +15,17 @@ describe('util/http/gitlab', () => { gitlabApi = new GitlabHttp(); setBaseUrl(`${gitlabApiHost}/api/v4/`); delete process.env.GITLAB_IGNORE_REPO_URL; + + hostRules.add({ + hostType: PLATFORM_TYPE_GITLAB, + token: 'abc123', + }); }); afterEach(() => { jest.resetAllMocks(); + + hostRules.clear(); }); it('paginates', async () => { @@ -68,6 +71,26 @@ describe('util/http/gitlab', () => { expect(trace).toHaveLength(3); expect(trace).toMatchSnapshot(); }); + + it('supports different datasources', async () => { + const gitlabApiDatasource = new GitlabHttp(GitlabReleasesDatasource.id); + hostRules.add({ hostType: PLATFORM_TYPE_GITLAB, token: 'abc' }); + hostRules.add({ + hostType: GitlabReleasesDatasource.id, + token: 'def', + }); + httpMock + .scope(gitlabApiHost, { reqheaders: { authorization: 'Bearer def' } }) + .get('/api/v4/some-url') + .reply(200); + const response = await gitlabApiDatasource.get('/some-url'); + expect(response).not.toBeNull(); + + const trace = httpMock.getTrace(); + expect(trace).toHaveLength(1); + expect(trace).toMatchSnapshot(); + }); + it('attempts to paginate', async () => { httpMock.scope(gitlabApiHost).get('/api/v4/some-url').reply(200, ['a'], { link: '; rel="last"', diff --git a/lib/util/http/gitlab.ts b/lib/util/http/gitlab.ts index a9ca70f00c883d..18eacdd2240f81 100644 --- a/lib/util/http/gitlab.ts +++ b/lib/util/http/gitlab.ts @@ -20,8 +20,11 @@ export interface GitlabHttpOptions extends InternalHttpOptions { } export class GitlabHttp extends Http { - constructor(options?: GitlabHttpOptions) { - super(PLATFORM_TYPE_GITLAB, options); + constructor( + type: string = PLATFORM_TYPE_GITLAB, + options?: GitlabHttpOptions + ) { + super(type, options); } protected override async request( diff --git a/lib/util/http/host-rules.ts b/lib/util/http/host-rules.ts index 557d44db75818a..975a2ae5412871 100644 --- a/lib/util/http/host-rules.ts +++ b/lib/util/http/host-rules.ts @@ -6,14 +6,20 @@ import { GotOptions } from './types'; // Apply host rules to requests export function applyHostRules(url: string, inOptions: GotOptions): GotOptions { - const options = { ...inOptions }; + const options: GotOptions = { ...inOptions }; const foundRules = hostRules.find({ hostType: options.hostType, url, }) || /* istanbul ignore next: can only happen in tests */ {}; const { username, password, token, enabled, authType } = foundRules; - if (options.headers?.authorization || options.password || options.token) { + if (options.noAuth) { + logger.trace({ url }, `Authorization disabled`); + } else if ( + options.headers?.authorization || + options.password || + options.token + ) { logger.trace({ url }, `Authorization already set`); } else if (password !== undefined) { logger.trace({ url }, `Applying Basic authentication`); diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 72a298434b0ecd..a3530c952606cb 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -26,6 +26,12 @@ export interface HttpOptions { password?: string; baseUrl?: string; headers?: OutgoingHttpHeaders; + + /** + * Do not use authentication + */ + noAuth?: boolean; + throwHttpErrors?: boolean; useCache?: boolean; } @@ -99,7 +105,11 @@ async function gotRoutine( } export class Http { - constructor(private hostType: string, private options?: HttpOptions) {} + private options?: GotOptions; + + constructor(private hostType: string, options?: HttpOptions) { + this.options = merge(options, { context: { hostType } }); + } protected async request( requestUrl: string | URL, @@ -136,13 +146,23 @@ export class Http { const cacheKey = crypto .createHash('md5') - .update('got-' + JSON.stringify({ url, headers: options.headers })) + .update( + 'got-' + + JSON.stringify({ + url, + headers: options.headers, + method: options.method, + }) + ) .digest('hex'); let resPromise; // Cache GET requests unless useCache=false - if (options.method === 'get' && options.useCache !== false) { + if ( + ['get', 'head'].includes(options.method) && + options.useCache !== false + ) { resPromise = memCache.get(cacheKey); } diff --git a/lib/util/http/types.ts b/lib/util/http/types.ts index c3741feeff70b9..b15287cfb558ef 100644 --- a/lib/util/http/types.ts +++ b/lib/util/http/types.ts @@ -20,6 +20,7 @@ export type GotExtraOptions = { hostType?: string; enabled?: boolean; useCache?: boolean; + noAuth?: boolean; context?: GotContextOptions; }; diff --git a/lib/util/url.spec.ts b/lib/util/url.spec.ts index c425ef9eb7a58c..d6f959e8b9a74d 100644 --- a/lib/util/url.spec.ts +++ b/lib/util/url.spec.ts @@ -1,5 +1,6 @@ import { ensurePathPrefix, + joinUrlParts, parseUrl, resolveBaseUrl, trimTrailingSlash, @@ -89,4 +90,20 @@ describe('util/url', () => { ensurePathPrefix('https://index.docker.io/v2/something', '/v2') ).toBe('https://index.docker.io/v2/something'); }); + + it('joinUrlParts', () => { + const registryUrl = 'https://some.test'; + expect(joinUrlParts(registryUrl, 'foo')).toBe(`${registryUrl}/foo`); + expect(joinUrlParts(registryUrl, '/?foo')).toBe(`${registryUrl}?foo`); + expect(joinUrlParts(registryUrl, '/foo/bar/')).toBe( + `${registryUrl}/foo/bar/` + ); + expect(joinUrlParts(`${registryUrl}/foo/`, '/foo/bar')).toBe( + `${registryUrl}/foo/foo/bar` + ); + expect(joinUrlParts(`${registryUrl}/api/`, '/foo/bar')).toBe( + `${registryUrl}/api/foo/bar` + ); + expect(joinUrlParts('foo//////')).toBe('foo/'); + }); }); diff --git a/lib/util/url.ts b/lib/util/url.ts index b84548b24138a1..ea9e560eb27b01 100644 --- a/lib/util/url.ts +++ b/lib/util/url.ts @@ -1,5 +1,9 @@ import urlJoin from 'url-join'; +export function joinUrlParts(...parts: string[]): string { + return urlJoin(...parts); +} + export function ensurePathPrefix(url: string, prefix: string): string { const parsed = new URL(url); const fullPath = url.replace(parsed.origin, ''); diff --git a/lib/versioning/docker/index.spec.ts b/lib/versioning/docker/index.spec.ts index c0248646e162e0..d9284a79c95484 100644 --- a/lib/versioning/docker/index.spec.ts +++ b/lib/versioning/docker/index.spec.ts @@ -2,128 +2,122 @@ import semver from '../semver'; import docker from '.'; describe('versioning/docker/index', () => { - describe('isValid(version)', () => { - it('should support all versions length', () => { - expect(docker.isValid(null)).toBeNull(); - expect(docker.isValid('1.2.3')).toBe('1.2.3'); - expect(docker.isValid('18.04')).toBe('18.04'); - expect(docker.isValid('10.1')).toBe('10.1'); - expect(docker.isValid('3')).toBe('3'); - expect(docker.isValid('foo')).toBeNull(); - }); - it('should return null if the version string looks like a git commit hash', () => { - [ - '0a1b2c3', - '0a1b2c3d', - '0a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d', - ].forEach((version) => { - expect(docker.isValid(version)).toBeNull(); - }); - [ - '0a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d0', - '0a1b2C3', - '0z1b2c3', - '0A1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d', - '123098140293', - ].forEach((version) => { - expect(docker.isValid(version)).toBe(version); - }); - }); - }); - describe('getMajor(version)', () => { - it('should support all versions length', () => { - expect(docker.getMajor('1.2.3')).toBe(1); - expect(docker.getMajor('18.04')).toBe(18); - expect(docker.getMajor('10.1')).toBe(10); - expect(docker.getMajor('3')).toBe(3); - expect(docker.getMajor('foo')).toBeNull(); - }); - }); - describe('getMinor(version)', () => { - it('should support all versions length', () => { - expect(docker.getMinor('1.2.3')).toBe(2); - expect(docker.getMinor('18.04')).toBe(4); - expect(docker.getMinor('10.1')).toBe(1); - expect(docker.getMinor('3')).toBeNull(); - expect(docker.getMinor('foo')).toBeNull(); - }); - }); - describe('getPatch(version)', () => { - it('should support all versions length', () => { - expect(docker.getPatch('1.2.3')).toBe(3); - expect(docker.getPatch('18.04')).toBeNull(); - expect(docker.getPatch('10.1')).toBeNull(); - expect(docker.getPatch('3')).toBeNull(); - expect(docker.getPatch('foo')).toBeNull(); - }); + test.each` + version | expected + ${null} | ${false} + ${'1.2.3'} | ${true} + ${'18.04'} | ${true} + ${'10.1'} | ${true} + ${'3'} | ${true} + ${'foo'} | ${false} + ${'0a1b2c3'} | ${false} + ${'0a1b2c3d'} | ${false} + ${'0a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d'} | ${false} + ${'0a1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d0'} | ${true} + ${'0a1b2C3'} | ${true} + ${'0z1b2c3'} | ${true} + ${'0A1b2c3d4e5f6a7b8c9d0a1b2c3d4e5f6a7b8c9d'} | ${true} + ${'123098140293'} | ${true} + `('isValid("$version") === $expected', ({ version, expected }) => { + const res = docker.isValid(version); + expect(!!res).toBe(expected); }); - describe('isGreaterThan(version, other)', () => { - it('should support all versions length', () => { - expect(docker.isGreaterThan('1.2.3', '1.2')).toBe(false); - expect(docker.isGreaterThan('18.04', '18.1')).toBe(true); - expect(docker.isGreaterThan('10.1', '10.1.2')).toBe(true); - expect(docker.isGreaterThan('3', '2')).toBe(true); - expect(docker.isGreaterThan('1.2.3', '1.2.3')).toBe(false); - }); - }); - describe('isLessThanRange(version, range)', () => { - it('should support all versions length', () => { - expect(docker.isLessThanRange('1.2.3', '2.0')).toBe(true); - expect(docker.isLessThanRange('18.04', '18.1')).toBe(false); - expect(docker.isLessThanRange('10.1', '10.0.4')).toBe(false); - expect(docker.isLessThanRange('3', '4.0')).toBe(true); - expect(docker.isLessThanRange('1.2', '1.3.4')).toBe(true); - }); + test.each` + version | major | minor | patch + ${'1.2.3'} | ${1} | ${2} | ${3} + ${'18.04'} | ${18} | ${4} | ${null} + ${'10.1'} | ${10} | ${1} | ${null} + ${'3'} | ${3} | ${null} | ${null} + ${'foo'} | ${null} | ${null} | ${null} + `( + 'getMajor, getMinor, getPatch for "$version"', + ({ version, major, minor, patch }) => { + expect(docker.getMajor(version)).toBe(major); + expect(docker.getMinor(version)).toBe(minor); + expect(docker.getPatch(version)).toBe(patch); + } + ); + + test.each` + a | b | expected + ${'1.2.3'} | ${'1.2'} | ${false} + ${'18.04'} | ${'18.1'} | ${true} + ${'10.1'} | ${'10.1.2'} | ${true} + ${'3'} | ${'2'} | ${true} + ${'1.2.3'} | ${'1.2.3'} | ${false} + `('isGreaterThan($a, $b) === $expected', ({ a, b, expected }) => { + expect(docker.isGreaterThan(a, b)).toBe(expected); }); - describe('equals(version, other)', () => { - it('should support all versions length', () => { - expect(docker.equals('1.2.3', '1.2.3')).toBe(true); - expect(docker.equals('18.04', '18.4')).toBe(true); - expect(docker.equals('10.0', '10.0.4')).toBe(false); - expect(docker.equals('3', '4.0')).toBe(false); - expect(docker.equals('1.2', '1.2.3')).toBe(false); - }); + + test.each` + version | range | expected + ${'1.2.3'} | ${'2.0'} | ${true} + ${'18.04'} | ${'18.1'} | ${false} + ${'10.1'} | ${'10.0.4'} | ${false} + ${'3'} | ${'4.0'} | ${true} + ${'1.2'} | ${'1.3.4'} | ${true} + `( + 'isLessThanRange($version, $range) === $expected', + ({ version, range, expected }) => { + expect(docker.isLessThanRange(version, range)).toBe(expected); + } + ); + + test.each` + a | b | expected + ${'1.2.3'} | ${'1.2.3'} | ${true} + ${'18.04'} | ${'18.4'} | ${true} + ${'10.0'} | ${'10.0.4'} | ${false} + ${'3'} | ${'4.0'} | ${false} + ${'1.2'} | ${'1.2.3'} | ${false} + `('equals($a, $b) === $expected', ({ a, b, expected }) => { + expect(docker.equals(a, b)).toBe(expected); }); - describe('getSatisfyingVersion(versions, range)', () => { - it('should support all versions length', () => { - [docker.minSatisfyingVersion, docker.getSatisfyingVersion].forEach( - (max) => { - const versions = [ - '0.9.8', - '1.1.1', - '1.1', - '1.2.3', - '1.2', - '1', - '2.2.2', - '2.2', - '2', - ]; - // returns range if found - expect(max(versions, '1.2.3')).toBe('1.2.3'); - expect(max(versions, '1.2')).toBe('1.2'); - expect(max(versions, '1')).toBe('1'); - // return null if not found - expect(max(versions, '1.3')).toBeNull(); - expect(max(versions, '0.9')).toBeNull(); - } - ); + + describe('Satisfying versions', () => { + const versions = [ + '0.9.8', + '1.1.1', + '1.1', + '1.2.3', + '1.2', + '1', + '2.2.2', + '2.2', + '2', + ]; + + test.each` + version | expected + ${'1.2.3'} | ${'1.2.3'} + ${'1.2'} | ${'1.2'} + ${'1'} | ${'1'} + ${'1.3'} | ${null} + ${'0.9'} | ${null} + `(`satisfying for $version -> $expected`, ({ version, expected }) => { + const satisfying = docker.getSatisfyingVersion(versions, version); + const minSatisfying = docker.minSatisfyingVersion(versions, version); + expect(satisfying).toBe(expected); + expect(minSatisfying).toBe(expected); }); }); + describe('sortVersions(v1, v2)', () => { - it('behaves like semver.sortVersions', () => { - [ - ['1.1.1', '1.2.3'], - ['1.2.3', '1.3.4'], - ['2.0.1', '1.2.3'], - ['1.2.3', '0.9.5'], - ].forEach((pair) => { - expect(docker.sortVersions(pair[0], pair[1])).toBe( - semver.sortVersions(pair[0], pair[1]) - ); - }); - }); + test.each` + a | b + ${'1.1.1'} | ${'1.2.3'} + ${'1.2.3'} | ${'1.3.4'} + ${'2.0.1'} | ${'1.2.3'} + ${'1.2.3'} | ${'0.9.5'} + `( + 'docker.sortVersions("$a", "$b") === semver.sortVersions("$a", "$b")', + ({ a, b }) => { + const dockerSorted = docker.sortVersions(a, b); + const semverSorted = semver.sortVersions(a, b); + expect(dockerSorted).toBe(semverSorted); + } + ); it('sorts unstable', () => { const versions = [ @@ -149,77 +143,69 @@ describe('versioning/docker/index', () => { ]); }); }); - describe('getNewValue(', () => { - it('returns newVersion', () => { - expect( - docker.getNewValue({ - currentValue: null, - rangeStrategy: null, - currentVersion: null, - newVersion: '1.2.3', - }) - ).toBe('1.2.3'); - }); - }); - - it('isStable(version)', () => { - const versions = [ - '3.7.0', - '3.7.0b1', - '3.7-alpine', - '3.8.0-alpine', - '3.8.0b1-alpine', - '3.8.2', - ]; - - expect(versions.filter(docker.isStable)).toEqual([ - '3.7.0', - '3.7-alpine', - '3.8.0-alpine', - '3.8.2', - ]); - }); - it('isCompatible(version)', () => { - const versions = [ - '3.7.0', - '3.7.0b1', - '3.7-alpine', - '3.8.0-alpine', - '3.8.0b1-alpine', - '3.8.2', - ]; - - expect(versions.filter((v) => docker.isCompatible(v, '3.7.0'))).toEqual([ - '3.7.0', - '3.7.0b1', - '3.8.2', - ]); + test.each` + currentValue | rangeStrategy | currentVersion | newVersion | expected + ${null} | ${null} | ${null} | ${'1.2.3'} | ${'1.2.3'} + `( + 'getNewValue($currentValue, $rangeStrategy, $currentVersion, $newVersion, $expected) === $expected', + ({ currentValue, rangeStrategy, currentVersion, newVersion, expected }) => { + const res = docker.getNewValue({ + currentValue, + rangeStrategy, + currentVersion, + newVersion, + }); + expect(res).toBe(expected); + } + ); - expect( - versions.filter((v) => docker.isCompatible(v, '3.7.0-alpine')) - ).toEqual(['3.8.0-alpine', '3.8.0b1-alpine']); + test.each` + version | expected + ${'3.7.0'} | ${true} + ${'3.7.0b1'} | ${false} + ${'3.7-alpine'} | ${true} + ${'3.8.0-alpine'} | ${true} + ${'3.8.0b1-alpine'} | ${false} + ${'3.8.2'} | ${true} + `('isStable("$version") === $expected', ({ version, expected }) => { + const res = docker.isStable(version); + expect(!!res).toBe(expected); }); - it('valueToVersion(version)', () => { - const versions = [ - '3.7.0', - '3.7.0b1', - '3.7-alpine', - '3.8.0-alpine', - '3.8.0b1-alpine', - '3.8.2', - undefined, - ]; + test.each` + version | range | expected + ${'3.7.0'} | ${'3.7.0'} | ${true} + ${'3.7.0b1'} | ${'3.7.0'} | ${true} + ${'3.7-alpine'} | ${'3.7.0'} | ${false} + ${'3.8.0-alpine'} | ${'3.7.0'} | ${false} + ${'3.8.0b1-alpine'} | ${'3.7.0'} | ${false} + ${'3.8.2'} | ${'3.7.0'} | ${true} + ${'3.7.0'} | ${'3.7.0-alpine'} | ${false} + ${'3.7.0b1'} | ${'3.7.0-alpine'} | ${false} + ${'3.7-alpine'} | ${'3.7.0-alpine'} | ${false} + ${'3.8.0-alpine'} | ${'3.7.0-alpine'} | ${true} + ${'3.8.0b1-alpine'} | ${'3.7.0-alpine'} | ${true} + ${'3.8.2'} | ${'3.7.0-alpine'} | ${false} + `( + 'isCompatible("$version") === $expected', + ({ version, range, expected }) => { + const res = docker.isCompatible(version, range); + expect(!!res).toBe(expected); + } + ); - expect(versions.map(docker.valueToVersion)).toEqual([ - '3.7.0', - '3.7.0b1', - '3.7', - '3.8.0', - '3.8.0b1', - '3.8.2', - undefined, - ]); + test.each` + value | expected + ${'3.7.0'} | ${'3.7.0'} + ${'3.7.0b1'} | ${'3.7.0b1'} + ${'3.7-alpine'} | ${'3.7'} + ${'3.8.0-alpine'} | ${'3.8.0'} + ${'3.8.0b1-alpine'} | ${'3.8.0b1'} + ${'3.8.2'} | ${'3.8.2'} + ${undefined} | ${undefined} + `('valueToVersion("$value") === $expected', ({ value, expected }) => { + const res = docker.valueToVersion(value); + expect(res).toBe(expected); }); }); diff --git a/lib/workers/pr/changelog/__snapshots__/github.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/github.spec.ts.snap index 23bcff4cd30e39..a97678ef16b0b3 100644 --- a/lib/workers/pr/changelog/__snapshots__/github.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/github.spec.ts.snap @@ -7,9 +7,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "@renovate/no", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -59,9 +60,10 @@ Object { "apiBaseUrl": "https://github-enterprise.example.com/api/v3/", "baseUrl": "https://github-enterprise.example.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github-enterprise.example.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github-enterprise.example.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -111,9 +113,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -163,9 +166,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -215,9 +219,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -267,9 +272,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { diff --git a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap index 427d3e91992835..cc229009feb0d3 100644 --- a/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/gitlab.spec.ts.snap @@ -7,8 +7,9 @@ Object { "apiBaseUrl": "https://gitlab.com/api/v4/", "baseUrl": "https://gitlab.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://gitlab.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://gitlab.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { @@ -154,8 +155,9 @@ Object { "apiBaseUrl": "https://gitlab-enterprise.example.com/api/v4/", "baseUrl": "https://gitlab-enterprise.example.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://gitlab-enterprise.example.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://gitlab-enterprise.example.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { @@ -197,8 +199,9 @@ Object { "apiBaseUrl": "https://git.test.com/api/v4/", "baseUrl": "https://git.test.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://git.test.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://git.test.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { @@ -240,8 +243,9 @@ Object { "apiBaseUrl": "https://gitlab.com/api/v4/", "baseUrl": "https://gitlab.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://gitlab.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://gitlab.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { @@ -403,8 +407,9 @@ Object { "apiBaseUrl": "https://gitlab.com/api/v4/", "baseUrl": "https://gitlab.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://gitlab.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://gitlab.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { @@ -550,8 +555,9 @@ Object { "apiBaseUrl": "https://gitlab.com/api/v4/", "baseUrl": "https://gitlab.com/", "depName": "renovate", - "gitlab": "meno/dropzone", - "repository": "https://gitlab.com/meno/dropzone/", + "repository": "meno/dropzone", + "sourceUrl": "https://gitlab.com/meno/dropzone/", + "type": "gitlab", }, "versions": Array [ Object { diff --git a/lib/workers/pr/changelog/__snapshots__/index.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/index.spec.ts.snap index 96be5e64cf56fd..f57ff11e1bfdda 100644 --- a/lib/workers/pr/changelog/__snapshots__/index.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/index.spec.ts.snap @@ -7,9 +7,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "@renovate/no", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -163,9 +164,10 @@ Object { "apiBaseUrl": "https://github-enterprise.example.com/api/v3/", "baseUrl": "https://github-enterprise.example.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github-enterprise.example.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github-enterprise.example.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -363,9 +365,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -563,9 +566,10 @@ Object { "apiBaseUrl": "https://github-enterprise.example.com/api/v3/", "baseUrl": "https://github-enterprise.example.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github-enterprise.example.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github-enterprise.example.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -763,9 +767,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -815,9 +820,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { @@ -1019,9 +1025,10 @@ Object { "apiBaseUrl": "https://api.github.com/", "baseUrl": "https://github.com/", "depName": "renovate", - "github": "chalk/chalk", - "repository": "https://github.com/chalk/chalk", + "repository": "chalk/chalk", "sourceDirectory": undefined, + "sourceUrl": "https://github.com/chalk/chalk", + "type": "github", }, "versions": Array [ Object { diff --git a/lib/workers/pr/changelog/__snapshots__/release-notes.spec.ts.snap b/lib/workers/pr/changelog/__snapshots__/release-notes.spec.ts.snap index 53776c97427260..976f87b5a518a1 100644 --- a/lib/workers/pr/changelog/__snapshots__/release-notes.spec.ts.snap +++ b/lib/workers/pr/changelog/__snapshots__/release-notes.spec.ts.snap @@ -5,7 +5,8 @@ Object { "a": 1, "hasReleaseNotes": false, "project": Object { - "github": "https://github.com/nodeca/js-yaml", + "repository": "https://github.com/nodeca/js-yaml", + "type": "github", }, "versions": Array [ Object { @@ -24,7 +25,8 @@ Object { "a": 1, "hasReleaseNotes": false, "project": Object { - "gitlab": "https://gitlab.com/gitlab-org/gitter/webapp/", + "repository": "https://gitlab.com/gitlab-org/gitter/webapp/", + "type": "gitlab", }, "versions": Array [ Object { @@ -137,7 +139,7 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 1`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "" 1`] = ` Object { "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) ", @@ -148,7 +150,7 @@ Object { } `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 2`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "" 2`] = ` Array [ Object { "headers": Object { @@ -163,18 +165,18 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 3`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other@" 1`] = ` Object { "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) ", "id": undefined, "name": undefined, - "tag": "v1.0.1", - "url": "https://github.com/some/other-repository/releases/v1.0.1", + "tag": "other@1.0.1", + "url": "https://github.com/some/other-repository/releases/other@1.0.1", } `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 4`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other@" 2`] = ` Array [ Object { "headers": Object { @@ -189,18 +191,18 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 5`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other_v" 1`] = ` Object { "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) ", "id": undefined, "name": undefined, - "tag": "other-1.0.1", - "url": "https://github.com/some/other-repository/releases/other-1.0.1", + "tag": "other_v1.0.1", + "url": "https://github.com/some/other-repository/releases/other_v1.0.1", } `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 6`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other_v" 2`] = ` Array [ Object { "headers": Object { @@ -215,18 +217,18 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 7`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other-" 1`] = ` Object { "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) ", "id": undefined, "name": undefined, - "tag": "other_v1.0.1", - "url": "https://github.com/some/other-repository/releases/other_v1.0.1", + "tag": "other-1.0.1", + "url": "https://github.com/some/other-repository/releases/other-1.0.1", } `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 8`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "other-" 2`] = ` Array [ Object { "headers": Object { @@ -241,18 +243,18 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 9`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "v" 1`] = ` Object { "body": "some body [#123](https://github.com/some/other-repository/issues/123), [#124](https://github.com/some/yet-other-repository/issues/124) ", "id": undefined, "name": undefined, - "tag": "other@1.0.1", - "url": "https://github.com/some/other-repository/releases/other@1.0.1", + "tag": "v1.0.1", + "url": "https://github.com/some/other-repository/releases/v1.0.1", } `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body 10`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body "v" 2`] = ` Array [ Object { "headers": Object { @@ -267,9 +269,16 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo 1`] = `null`; +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "" 1`] = ` +Object { + "body": "some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)", + "name": undefined, + "tag": "1.0.1", + "url": "https://gitlab.com/some/other-repository/tags/1.0.1", +} +`; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo 2`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "" 2`] = ` Array [ Object { "headers": Object { @@ -284,9 +293,16 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo other- 1`] = `null`; +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "other-" 1`] = ` +Object { + "body": "some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)", + "name": undefined, + "tag": "other-1.0.1", + "url": "https://gitlab.com/some/other-repository/tags/other-1.0.1", +} +`; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo other- 2`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "other-" 2`] = ` Array [ Object { "headers": Object { @@ -301,9 +317,16 @@ Array [ ] `; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo v 1`] = `null`; +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "v" 1`] = ` +Object { + "body": "some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)", + "name": undefined, + "tag": "v1.0.1", + "url": "https://gitlab.com/some/other-repository/tags/v1.0.1", +} +`; -exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo v 2`] = ` +exports[`workers/pr/changelog/release-notes getReleaseNotes() gets release notes with body from gitlab repo "v" 2`] = ` Array [ Object { "headers": Object { diff --git a/lib/workers/pr/changelog/github/index.ts b/lib/workers/pr/changelog/github/index.ts index 4232aa7550245f..354e7da27d6308 100644 --- a/lib/workers/pr/changelog/github/index.ts +++ b/lib/workers/pr/changelog/github/index.ts @@ -1,4 +1,6 @@ import changelogFilenameRegex from 'changelog-filename-regex'; +import type { GithubRelease } from '../../../../datasource/github-releases/types'; +import type { GitHubTag } from '../../../../datasource/github-tags/types'; import { logger } from '../../../../logger'; import type { GithubGitBlob, @@ -18,7 +20,7 @@ export async function getTags( logger.trace('github.getTags()'); const url = `${endpoint}repos/${repository}/tags?per_page=100`; try { - const res = await http.getJson<{ name: string }[]>(url, { + const res = await http.getJson(url, { paginate: true, }); @@ -30,8 +32,10 @@ export async function getTags( return tags.map((tag) => tag.name).filter(Boolean); } catch (err) { - logger.debug({ sourceRepo: repository }, 'Failed to fetch Github tags'); - logger.debug({ err }); + logger.debug( + { sourceRepo: repository, err }, + 'Failed to fetch Github tags' + ); // istanbul ignore if if (err.message?.includes('Bad credentials')) { logger.warn('Bad credentials triggering tag fail lookup in changelog'); @@ -108,15 +112,7 @@ export async function getReleaseList( const url = `${ensureTrailingSlash( apiBaseUrl )}repos/${repository}/releases?per_page=100`; - const res = await http.getJson< - { - html_url: string; - id: number; - tag_name: string; - name: string; - body: string; - }[] - >(url, { paginate: true }); + const res = await http.getJson(url, { paginate: true }); return res.body.map((release) => ({ url: release.html_url, id: release.id, diff --git a/lib/workers/pr/changelog/gitlab/index.ts b/lib/workers/pr/changelog/gitlab/index.ts index d6a0b94fd54676..ab7e71625d4f12 100644 --- a/lib/workers/pr/changelog/gitlab/index.ts +++ b/lib/workers/pr/changelog/gitlab/index.ts @@ -1,4 +1,6 @@ import changelogFilenameRegex from 'changelog-filename-regex'; +import type { GitlabRelease } from '../../../../datasource/gitlab-releases/types'; +import type { GitlabTag } from '../../../../datasource/gitlab-tags/types'; import { logger } from '../../../../logger'; import type { GitlabTreeNode } from '../../../../types/platform/gitlab'; import { GitlabHttp } from '../../../../util/http/gitlab'; @@ -20,7 +22,7 @@ export async function getTags( repository )}/repository/tags?per_page=100`; try { - const res = await http.getJson<{ name: string }[]>(url, { + const res = await http.getJson(url, { paginate: true, }); @@ -32,7 +34,10 @@ export async function getTags( return tags.map((tag) => tag.name).filter(Boolean); } catch (err) { - logger.info({ sourceRepo: repository }, 'Failed to fetch Gitlab tags'); + logger.debug( + { sourceRepo: repository, err }, + 'Failed to fetch Gitlab tags' + ); // istanbul ignore if if (err.message?.includes('Bad credentials')) { logger.warn('Bad credentials triggering tag fail lookup in changelog'); @@ -101,14 +106,8 @@ export async function getReleaseList( const apiUrl = `${ensureTrailingSlash( apiBaseUrl )}projects/${repoId}/releases`; - const res = await http.getJson< - { - name: string; - release: string; - description: string; - tag_name: string; - }[] - >(`${apiUrl}?per_page=100`, { + + const res = await http.getJson(`${apiUrl}?per_page=100`, { paginate: true, }); return res.body.map((release) => ({ diff --git a/lib/workers/pr/changelog/release-notes.spec.ts b/lib/workers/pr/changelog/release-notes.spec.ts index 526e23d1901968..3a86c3db889fb9 100644 --- a/lib/workers/pr/changelog/release-notes.spec.ts +++ b/lib/workers/pr/changelog/release-notes.spec.ts @@ -10,7 +10,12 @@ import { getReleaseNotesMd, releaseNotesCacheMinutes, } from './release-notes'; -import type { ChangeLogNotes } from './types'; +import type { + ChangeLogNotes, + ChangeLogProject, + ChangeLogRelease, + ChangeLogResult, +} from './types'; jest.mock('../../../util/host-rules'); @@ -37,6 +42,18 @@ const gitlabTreeResponse = [ { path: 'README.md', type: 'blob' }, ]; +const githubProject = { + type: 'github', + apiBaseUrl: 'https://api.github.com/', + baseUrl: 'https://github.com/', +} as ChangeLogProject; + +const gitlabProject = { + type: 'gitlab', + apiBaseUrl: 'https://gitlab.com/api/v4/', + baseUrl: 'https://gitlab.com/', +} as ChangeLogProject; + describe('workers/pr/changelog/release-notes', () => { beforeEach(() => { hostRules.find.mockReturnValue({}); @@ -60,6 +77,7 @@ describe('workers/pr/changelog/release-notes', () => { it('handles date object', () => { expect(releaseNotesCacheMinutes(new Date())).toEqual(55); }); + it.each([null, undefined, 'fake', 123])('handles invalid: %s', (date) => { expect(releaseNotesCacheMinutes(date as never)).toEqual(55); }); @@ -69,31 +87,47 @@ describe('workers/pr/changelog/release-notes', () => { it('returns input if invalid', async () => { const input = { a: 1 }; expect(await addReleaseNotes(input as never)).toEqual(input); + expect(await addReleaseNotes(null)).toBeNull(); + expect(await addReleaseNotes({ versions: [] } as never)).toStrictEqual({ + versions: [], + }); }); + it('returns ChangeLogResult', async () => { const input = { a: 1, - project: { github: 'https://github.com/nodeca/js-yaml' }, + project: { + type: 'github', + repository: 'https://github.com/nodeca/js-yaml', + }, versions: [{ version: '3.10.0', compare: { url: '' } }], }; // FIXME: explicit assert condition expect(await addReleaseNotes(input as never)).toMatchSnapshot(); }); + it('returns ChangeLogResult without release notes', async () => { const input = { a: 1, - project: { gitlab: 'https://gitlab.com/gitlab-org/gitter/webapp/' }, - versions: [{ version: '20.26.0', compare: { url: '' } }], - }; + project: { + type: 'gitlab', + repository: 'https://gitlab.com/gitlab-org/gitter/webapp/', + } as ChangeLogProject, + versions: [ + { version: '20.26.0', compare: { url: '' } } as ChangeLogRelease, + ], + } as ChangeLogResult; // FIXME: explicit assert condition - expect(await addReleaseNotes(input as never)).toMatchSnapshot(); + expect(await addReleaseNotes(input)).toMatchSnapshot(); }); }); + describe('getReleaseList()', () => { it('should return empty array if no apiBaseUrl', async () => { - const res = await getReleaseList('', 'some/yet-other-repository'); + const res = await getReleaseList({} as ChangeLogProject); expect(res).toEqual([]); }); + it('should return release list for github repo', async () => { httpMock .scope('https://api.github.com/') @@ -106,14 +140,15 @@ describe('workers/pr/changelog/release-notes', () => { }, ]); - const res = await getReleaseList( - 'https://api.github.com/', - 'some/yet-other-repository' - ); + const res = await getReleaseList({ + ...githubProject, + repository: 'some/yet-other-repository', + }); // FIXME: explicit assert condition expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('should return release list for gitlab.com project', async () => { httpMock .scope('https://gitlab.com/') @@ -127,14 +162,15 @@ describe('workers/pr/changelog/release-notes', () => { body: 'some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)', }, ]); - const res = await getReleaseList( - 'https://gitlab.com/api/v4/', - 'some/yet-other-repository' - ); + const res = await getReleaseList({ + ...gitlabProject, + repository: 'some/yet-other-repository', + }); // FIXME: explicit assert condition expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('should return release list for self hosted gitlab project', async () => { hostRules.find.mockReturnValue({ token: 'some-token' }); httpMock @@ -149,15 +185,18 @@ describe('workers/pr/changelog/release-notes', () => { body: 'some body #123, [#124](https://my.custom.domain/some/yet-other-repository/issues/124)', }, ]); - const res = await getReleaseList( - 'https://my.custom.domain/api/v4/', - 'some/yet-other-repository' - ); + const res = await getReleaseList({ + ...gitlabProject, + repository: 'some/yet-other-repository', + apiBaseUrl: 'https://my.custom.domain/api/v4/', + baseUrl: 'https://my.custom.domain/', + }); // FIXME: explicit assert condition expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); }); + describe('getReleaseNotes()', () => { it('should return null for release notes without body', async () => { httpMock @@ -165,17 +204,19 @@ describe('workers/pr/changelog/release-notes', () => { .get('/repos/some/repository/releases?per_page=100') .reply(200, [{ tag_name: 'v1.0.0' }, { tag_name: 'v1.0.1' }]); const res = await getReleaseNotes( - 'some/repository', - '1.0.0', - 'some', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'some/repository', + depName: 'some', + }, + '1.0.0' ); expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it.each([[''], ['v'], ['other-'], ['other_v'], ['other@']])( - 'gets release notes with body', + 'gets release notes with body "%s"', async (prefix) => { httpMock .scope('https://api.github.com/') @@ -188,19 +229,21 @@ describe('workers/pr/changelog/release-notes', () => { }, ]); const res = await getReleaseNotes( - 'some/other-repository', - '1.0.1', - 'other', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'some/other-repository', + depName: 'other', + }, + '1.0.1' ); // FIXME: explicit assert condition expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); } ); + it.each([[''], ['v'], ['other-']])( - 'gets release notes with body from gitlab repo %s', + 'gets release notes with body from gitlab repo "%s"', async (prefix) => { httpMock .scope('https://api.gitlab.com/') @@ -209,52 +252,53 @@ describe('workers/pr/changelog/release-notes', () => { { tag_name: `${prefix}1.0.0` }, { tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)', + description: + 'some body #123, [#124](https://gitlab.com/some/yet-other-repository/issues/124)', }, ]); const res = await getReleaseNotes( - 'some/other-repository', - '1.0.1', - 'other', - 'https://gitlab.com/', - 'https://api.gitlab.com/' + { + ...gitlabProject, + repository: 'some/other-repository', + depName: 'other', + apiBaseUrl: 'https://api.gitlab.com/', + }, + '1.0.1' ); // FIXME: explicit assert condition expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); } ); - it.each([[''], ['v'], ['other-']])( - 'gets null from repository without gitlab/github in domain %s', - async (prefix) => { - // FIXME: Should not call `api.lol.lol` ? - httpMock - .scope('https://api.lol.lol') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(404); - const res = await getReleaseNotes( - 'some/other-repository', - '1.0.1', - 'other', - 'https://lol.lol/', - 'https://api.lol.lol/' - ); - expect(res).toBeNull(); - } - ); + + it('gets null from repository without gitlab/github in domain', async () => { + const res = await getReleaseNotes( + { + repository: 'some/repository', + depName: 'other', + apiBaseUrl: 'https://api.lol.lol/', + baseUrl: 'https://lol.lol/', + } as ChangeLogProject, + '1.0.1' + ); + expect(res).toBeNull(); + }); }); + describe('getReleaseNotesMd()', () => { it('handles not found', async () => { httpMock.scope('https://api.github.com').get('/repos/chalk').reply(404); const res = await getReleaseNotesMd( - 'chalk', - '2.0.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'chalk', + }, + '2.0.0' ); expect(res).toBeNull(); }); + it('handles files mismatch', async () => { httpMock .scope('https://api.github.com') @@ -268,14 +312,16 @@ describe('workers/pr/changelog/release-notes', () => { ], }); const res = await getReleaseNotesMd( - 'chalk', - '2.0.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'chalk', + }, + '2.0.0' ); expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('handles wrong format', async () => { httpMock .scope('https://api.github.com') @@ -288,14 +334,16 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from('not really markdown').toString('base64'), }); const res = await getReleaseNotesMd( - 'some/repository1', - '1.0.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'some/repository1', + }, + '1.0.0' ); expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('handles bad markdown', async () => { httpMock .scope('https://api.github.com') @@ -308,14 +356,16 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(`#\nha\nha\n#\nha\nha`).toString('base64'), }); const res = await getReleaseNotesMd( - 'some/repository2', - '1.0.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'some/repository2', + }, + '1.0.0' ); expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('parses angular.js', async () => { httpMock .scope('https://api.github.com') @@ -328,16 +378,18 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(angularJsChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'angular/angular.js', - '1.6.9', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'angular/angular.js', + }, + '1.6.9' ); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('parses gitlab.com/gitlab-org/gitter/webapp', async () => { jest.setTimeout(0); httpMock @@ -349,16 +401,19 @@ describe('workers/pr/changelog/release-notes', () => { .get('/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw') .reply(200, gitterWebappChangelogMd); const res = await getReleaseNotesMd( - 'gitlab-org/gitter/webapp', - '20.26.0', - 'https://gitlab.com/', - 'https://api.gitlab.com/' + { + ...gitlabProject, + repository: 'gitlab-org/gitter/webapp', + apiBaseUrl: 'https://api.gitlab.com/', + }, + '20.26.0' ); expect(httpMock.getTrace()).toMatchSnapshot(); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('parses self hosted gitlab', async () => { hostRules.find.mockReturnValue({ token: 'some-token' }); jest.setTimeout(0); @@ -371,16 +426,20 @@ describe('workers/pr/changelog/release-notes', () => { .get('/projects/gitlab-org%2fgitter%2fwebapp/repository/blobs/abcd/raw') .reply(200, gitterWebappChangelogMd); const res = await getReleaseNotesMd( - 'gitlab-org/gitter/webapp', - '20.26.0', - 'https://my.custom.domain/', - 'https://my.custom.domain/' + { + ...gitlabProject, + repository: 'gitlab-org/gitter/webapp', + apiBaseUrl: 'https://my.custom.domain/', + baseUrl: 'https://my.custom.domain/', + }, + '20.26.0' ); expect(httpMock.getTrace()).toMatchSnapshot(); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('parses jest', async () => { httpMock .scope('https://api.github.com') @@ -393,16 +452,18 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(jestChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'facebook/jest', - '22.0.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'facebook/jest', + }, + '22.0.0' ); expect(httpMock.getTrace()).toMatchSnapshot(); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('handles github sourceDirectory', async () => { const sourceDirectory = 'packages/foo'; const subdirTree = clone(githubTreeResponse); @@ -420,17 +481,19 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(jsYamlChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'nodeca/js-yaml', - '3.10.0', - 'https://github.com/', - 'https://api.github.com/', - sourceDirectory + { + ...githubProject, + repository: 'nodeca/js-yaml', + sourceDirectory, + }, + '3.10.0' ); expect(httpMock.getTrace()).toMatchSnapshot(); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('parses js-yaml', async () => { httpMock .scope('https://api.github.com') @@ -443,16 +506,28 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(jsYamlChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'nodeca/js-yaml', - '3.10.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'nodeca/js-yaml', + }, + '3.10.0' ); expect(httpMock.getTrace()).toMatchSnapshot(); // FIXME: explicit assert condition expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + + it('ignores invalid', async () => { + const res = await getReleaseNotesMd( + { + repository: 'nodeca/js-yaml', + } as ChangeLogProject, + '3.10.0' + ); + expect(res).toBeNull(); + }); + describe('ReleaseNotes Correctness', () => { let versionOneNotes: ChangeLogNotes; let versionTwoNotes: ChangeLogNotes; @@ -468,10 +543,11 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(yargsChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'yargs/yargs', - '15.3.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'yargs/yargs', + }, + '15.3.0' ); versionOneNotes = res; expect(httpMock.getTrace()).toMatchSnapshot(); @@ -479,6 +555,7 @@ describe('workers/pr/changelog/release-notes', () => { expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('parses yargs 15.2.0', async () => { httpMock .scope('https://api.github.com') @@ -491,10 +568,11 @@ describe('workers/pr/changelog/release-notes', () => { content: Buffer.from(yargsChangelogMd).toString('base64'), }); const res = await getReleaseNotesMd( - 'yargs/yargs', - '15.2.0', - 'https://github.com/', - 'https://api.github.com/' + { + ...githubProject, + repository: 'yargs/yargs', + }, + '15.2.0' ); versionTwoNotes = res; expect(httpMock.getTrace()).toMatchSnapshot(); @@ -502,6 +580,7 @@ describe('workers/pr/changelog/release-notes', () => { expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('parses adapter-utils 4.33.0', async () => { httpMock .scope('https://gitlab.com/') @@ -514,10 +593,11 @@ describe('workers/pr/changelog/release-notes', () => { ) .reply(200, adapterutilsChangelogMd); const res = await getReleaseNotesMd( - 'itentialopensource/adapter-utils', - '4.33.0', - 'https://gitlab.com/', - 'https://gitlab.com/api/v4/' + { + ...gitlabProject, + repository: 'itentialopensource/adapter-utils', + }, + '4.33.0' ); versionTwoNotes = res; expect(httpMock.getTrace()).toMatchSnapshot(); @@ -525,6 +605,7 @@ describe('workers/pr/changelog/release-notes', () => { expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('handles gitlab sourceDirectory', async () => { const sourceDirectory = 'packages/foo'; const response = clone(gitlabTreeResponse).map((file) => ({ @@ -542,11 +623,12 @@ describe('workers/pr/changelog/release-notes', () => { ) .reply(200, adapterutilsChangelogMd); const res = await getReleaseNotesMd( - 'itentialopensource/adapter-utils', - '4.33.0', - 'https://gitlab.com/', - 'https://gitlab.com/api/v4/', - sourceDirectory + { + ...gitlabProject, + repository: 'itentialopensource/adapter-utils', + sourceDirectory, + }, + '4.33.0' ); versionTwoNotes = res; expect(httpMock.getTrace()).toMatchSnapshot(); @@ -554,9 +636,11 @@ describe('workers/pr/changelog/release-notes', () => { expect(res).not.toBeNull(); expect(res).toMatchSnapshot(); }); + it('isUrl', () => { expect(versionOneNotes).not.toMatchObject(versionTwoNotes); }); + it('15.3.0 is not equal to 15.2.0', () => { expect(versionOneNotes).not.toMatchObject(versionTwoNotes); }); diff --git a/lib/workers/pr/changelog/release-notes.ts b/lib/workers/pr/changelog/release-notes.ts index 51fba383c3805d..43067c97094d56 100644 --- a/lib/workers/pr/changelog/release-notes.ts +++ b/lib/workers/pr/changelog/release-notes.ts @@ -2,64 +2,61 @@ import * as URL from 'url'; import is from '@sindresorhus/is'; import { DateTime } from 'luxon'; import MarkdownIt from 'markdown-it'; -import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms'; import { logger } from '../../../logger'; import * as memCache from '../../../util/cache/memory'; import * as packageCache from '../../../util/cache/package'; -import * as hostRules from '../../../util/host-rules'; import { linkify } from '../../../util/markdown'; import * as github from './github'; import * as gitlab from './gitlab'; -import type { ChangeLogFile, ChangeLogNotes, ChangeLogResult } from './types'; +import type { + ChangeLogFile, + ChangeLogNotes, + ChangeLogProject, + ChangeLogResult, +} from './types'; const markdown = new MarkdownIt('zero'); markdown.enable(['heading', 'lheading']); export async function getReleaseList( - apiBaseUrl: string, - repository: string + project: ChangeLogProject ): Promise { logger.trace('getReleaseList()'); - // istanbul ignore if - if (!apiBaseUrl) { - logger.debug('No apiBaseUrl'); - return []; - } + const { apiBaseUrl, repository, type } = project; try { - if (apiBaseUrl.includes('gitlab')) { - return await gitlab.getReleaseList(apiBaseUrl, repository); - } + switch (type) { + case 'gitlab': + return await gitlab.getReleaseList(apiBaseUrl, repository); + case 'github': + return await github.getReleaseList(apiBaseUrl, repository); - const opts = hostRules.find({ - hostType: PLATFORM_TYPE_GITLAB, - url: apiBaseUrl, - }); - if (opts.token) { - return await gitlab.getReleaseList(apiBaseUrl, repository); + default: + logger.warn({ apiBaseUrl, repository, type }, 'Invalid project type'); + return []; } - - return await github.getReleaseList(apiBaseUrl, repository); } catch (err) /* istanbul ignore next */ { if (err.statusCode === 404) { - logger.debug({ repository }, 'getReleaseList 404'); + logger.debug({ repository, type, apiBaseUrl }, 'getReleaseList 404'); } else { - logger.info({ repository, err }, 'getReleaseList error'); + logger.debug( + { repository, type, apiBaseUrl, err }, + 'getReleaseList error' + ); } - return []; } + return []; } export function getCachedReleaseList( - apiBaseUrl: string, - repository: string + project: ChangeLogProject ): Promise { - const cacheKey = `getReleaseList-${apiBaseUrl}-${repository}`; + const cacheKey = `getReleaseList-${project.apiBaseUrl}-${project.repository}`; const cachedResult = memCache.get>(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } - const promisedRes = getReleaseList(apiBaseUrl, repository); + const promisedRes = getReleaseList(project); memCache.set(cacheKey, promisedRes); return promisedRes; } @@ -94,14 +91,12 @@ export function massageBody( } export async function getReleaseNotes( - repository: string, - version: string, - depName: string, - baseUrl: string, - apiBaseUrl: string + project: ChangeLogProject, + version: string ): Promise { + const { baseUrl, depName, repository } = project; logger.trace(`getReleaseNotes(${repository}, ${version}, ${depName})`); - const releaseList = await getCachedReleaseList(apiBaseUrl, repository); + const releaseList = await getCachedReleaseList(project); logger.trace({ releaseList }, 'Release list from getReleaseList'); let releaseNotes: ChangeLogNotes | null = null; for (const release of releaseList) { @@ -172,84 +167,70 @@ function isUrl(url: string): boolean { } export async function getReleaseNotesMdFileInner( - repository: string, - apiBaseUrl: string, - sourceDirectory?: string + project: ChangeLogProject ): Promise | null { + const { apiBaseUrl, repository, sourceDirectory, type } = project; try { - if (apiBaseUrl.includes('gitlab')) { - return await gitlab.getReleaseNotesMd( - repository, - apiBaseUrl, - sourceDirectory - ); - } + switch (type) { + case 'gitlab': + return await gitlab.getReleaseNotesMd( + repository, + apiBaseUrl, + sourceDirectory + ); + case 'github': + return await github.getReleaseNotesMd( + repository, + apiBaseUrl, + sourceDirectory + ); - const opts = hostRules.find({ - hostType: PLATFORM_TYPE_GITLAB, - url: apiBaseUrl, - }); - if (opts.token) { - return await gitlab.getReleaseNotesMd( - repository, - apiBaseUrl, - sourceDirectory - ); + default: + logger.warn({ apiBaseUrl, repository, type }, 'Invalid project type'); + return null; } - - return await github.getReleaseNotesMd( - repository, - apiBaseUrl, - sourceDirectory - ); } catch (err) /* istanbul ignore next */ { if (err.statusCode === 404) { - logger.debug('Error 404 getting changelog md'); + logger.debug( + { repository, type, apiBaseUrl }, + 'Error 404 getting changelog md' + ); } else { - logger.debug({ err, repository }, 'Error getting changelog md'); + logger.debug( + { err, repository, type, apiBaseUrl }, + 'Error getting changelog md' + ); } - return null; } + return null; } export function getReleaseNotesMdFile( - repository: string, - apiBaseUrl: string, - sourceDirectory?: string + project: ChangeLogProject ): Promise { - const cacheKey = `getReleaseNotesMdFile-${repository}-${apiBaseUrl}`; + const cacheKey = `getReleaseNotesMdFile-${project.repository}-${project.apiBaseUrl}`; const cachedResult = memCache.get>(cacheKey); // istanbul ignore if if (cachedResult !== undefined) { return cachedResult; } - const promisedRes = getReleaseNotesMdFileInner( - repository, - apiBaseUrl, - sourceDirectory - ); + const promisedRes = getReleaseNotesMdFileInner(project); memCache.set(cacheKey, promisedRes); return promisedRes; } export async function getReleaseNotesMd( - repository: string, - version: string, - baseUrl: string, - apiBaseUrl: string, - sourceDirectory?: string + project: ChangeLogProject, + version: string ): Promise { + const { baseUrl, repository } = project; logger.trace(`getReleaseNotesMd(${repository}, ${version})`); const skippedRepos = ['facebook/react-native']; // istanbul ignore if if (skippedRepos.includes(repository)) { return null; } - const changelog = await getReleaseNotesMdFile( - repository, - apiBaseUrl, - sourceDirectory - ); + const changelog = await getReleaseNotesMdFile(project); if (!changelog) { return null; } @@ -332,20 +313,13 @@ export function releaseNotesCacheMinutes(releaseDate?: string | Date): number { export async function addReleaseNotes( input: ChangeLogResult ): Promise { - if ( - !input?.versions || - (!input?.project?.github && !input?.project?.gitlab) - ) { + if (!input?.versions || !input.project?.type) { logger.debug('Missing project or versions'); return input; } const output: ChangeLogResult = { ...input, versions: [] }; - const repository = input.project.github - ? input.project.github.replace(/\.git$/, '') - : input.project.gitlab; - const cacheNamespace = input.project.github - ? 'changelog-github-notes' - : 'changelog-gitlab-notes'; + const repository = input.project.repository; + const cacheNamespace = `changelog-${input.project.type}-notes`; function getCacheKey(version: string): string { return `${repository}:${version}`; } @@ -355,23 +329,10 @@ export async function addReleaseNotes( releaseNotes = await packageCache.get(cacheNamespace, cacheKey); // istanbul ignore else: no cache tests if (!releaseNotes) { - const { sourceDirectory } = input.project; - releaseNotes = await getReleaseNotesMd( - repository, - v.version, - input.project.baseUrl, - input.project.apiBaseUrl, - sourceDirectory - ); + releaseNotes = await getReleaseNotesMd(input.project, v.version); // istanbul ignore else: should be tested if (!releaseNotes) { - releaseNotes = await getReleaseNotes( - repository, - v.version, - input.project.depName, - input.project.baseUrl, - input.project.apiBaseUrl - ); + releaseNotes = await getReleaseNotes(input.project, v.version); } // Small hack to force display of release notes when there is a compare url if (!releaseNotes && v.compare.url) { diff --git a/lib/workers/pr/changelog/source-github.ts b/lib/workers/pr/changelog/source-github.ts index b4577088b42161..3dee765d2e0825 100644 --- a/lib/workers/pr/changelog/source-github.ts +++ b/lib/workers/pr/changelog/source-github.ts @@ -68,7 +68,10 @@ export async function getChangeLogJSON({ const apiBaseUrl = sourceUrl.startsWith('https://github.com/') ? 'https://api.github.com/' : baseUrl + 'api/v3/'; - const repository = pathname.slice(1).replace(/\/$/, ''); + const repository = pathname + .slice(1) + .replace(/\/$/, '') + .replace(/\.git$/, ''); if (repository.split('/').length !== 2) { logger.debug({ sourceUrl }, 'Invalid github URL found'); return null; @@ -154,8 +157,9 @@ export async function getChangeLogJSON({ project: { apiBaseUrl, baseUrl, - github: repository, - repository: sourceUrl, + type: 'github', + repository, + sourceUrl, sourceDirectory, depName, }, diff --git a/lib/workers/pr/changelog/source-gitlab.ts b/lib/workers/pr/changelog/source-gitlab.ts index c63c91c4855b31..c241bb9a04f741 100644 --- a/lib/workers/pr/changelog/source-gitlab.ts +++ b/lib/workers/pr/changelog/source-gitlab.ts @@ -130,8 +130,9 @@ export async function getChangeLogJSON({ project: { apiBaseUrl, baseUrl, - gitlab: repository, - repository: sourceUrl, + type: 'gitlab', + repository, + sourceUrl, depName, }, versions: changelogReleases, diff --git a/lib/workers/pr/changelog/types.ts b/lib/workers/pr/changelog/types.ts index 554eefc2c4d04c..48217fc59eab65 100644 --- a/lib/workers/pr/changelog/types.ts +++ b/lib/workers/pr/changelog/types.ts @@ -22,11 +22,11 @@ export interface ChangeLogRelease { export interface ChangeLogProject { depName?: string; - github?: string; - gitlab?: string; + type: 'github' | 'gitlab'; apiBaseUrl?: string; baseUrl: string; repository: string; + sourceUrl: string; sourceDirectory?: string; } diff --git a/lib/workers/pr/index.spec.ts b/lib/workers/pr/index.spec.ts index 746c5dff8fe041..d82283479371cd 100644 --- a/lib/workers/pr/index.spec.ts +++ b/lib/workers/pr/index.spec.ts @@ -1,21 +1,18 @@ -import { git, mocked, partial } from '../../../test/util'; -import { getConfig } from '../../config/defaults'; +import { getConfig, git, mocked, partial, platform } from '../../../test/util'; import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; -import { Pr, platform as _platform } from '../../platform'; +import type { Pr } from '../../platform/types'; import { BranchStatus } from '../../types'; import * as _limits from '../global/limits'; import type { BranchConfig } from '../types'; import * as prAutomerge from './automerge'; import * as _changelogHelper from './changelog'; -import { getChangeLogJSON } from './changelog'; +import type { ChangeLogResult } from './changelog'; import * as codeOwners from './code-owners'; import * as prWorker from '.'; const codeOwnersMock = mocked(codeOwners); const changelogHelper = mocked(_changelogHelper); const gitlabChangelogHelper = mocked(_changelogHelper); -const platform = mocked(_platform); -const defaultConfig = getConfig(); const limits = mocked(_limits); jest.mock('../../util/git'); @@ -24,12 +21,12 @@ jest.mock('./code-owners'); jest.mock('../global/limits'); function setupChangelogMock() { - changelogHelper.getChangeLogJSON = jest.fn(); const resultValue = { project: { + type: 'github', baseUrl: 'https://github.com/', - github: 'renovateapp/dummy', - repository: 'https://github.com/renovateapp/dummy', + repository: 'renovateapp/dummy', + sourceUrl: 'https://github.com/renovateapp/dummy', }, hasReleaseNotes: true, versions: [ @@ -51,7 +48,7 @@ function setupChangelogMock() { }, }, ], - }; + } as ChangeLogResult; const errorValue = { error: _changelogHelper.ChangeLogError.MissingGithubToken, }; @@ -61,12 +58,12 @@ function setupChangelogMock() { } function setupGitlabChangelogMock() { - gitlabChangelogHelper.getChangeLogJSON = jest.fn(); const resultValue = { project: { + type: 'gitlab', baseUrl: 'https://gitlab.com/', - gitlab: 'renovateapp/gitlabdummy', - repository: 'https://gitlab.com/renovateapp/gitlabdummy', + repository: 'renovateapp/gitlabdummy', + sourceUrl: 'https://gitlab.com/renovateapp/gitlabdummy', }, hasReleaseNotes: true, versions: [ @@ -88,7 +85,7 @@ function setupGitlabChangelogMock() { }, }, ], - }; + } as ChangeLogResult; const errorValue = { error: _changelogHelper.ChangeLogError.MissingGithubToken, }; @@ -103,7 +100,7 @@ describe('workers/pr/index', () => { let pr: Pr; beforeEach(() => { config = partial({ - ...defaultConfig, + ...getConfig(), }); pr = partial({ canMerge: true, @@ -180,7 +177,7 @@ describe('workers/pr/index', () => { jest.resetAllMocks(); setupChangelogMock(); config = partial({ - ...defaultConfig, + ...getConfig(), }); config.branchName = 'renovate/dummy-1.x'; config.prTitle = 'Update dependency dummy to v1.1.0'; @@ -199,9 +196,7 @@ describe('workers/pr/index', () => { displayNumber: 'New Pull Request', } as never); config.upgrades = [config]; - platform.massageMarkdown = jest.fn((input) => input); - platform.getBranchPr = jest.fn(); - platform.getBranchStatus = jest.fn(); + platform.massageMarkdown.mockImplementation((input) => input); }); afterEach(() => { jest.clearAllMocks(); @@ -252,7 +247,7 @@ describe('workers/pr/index', () => { }); it('should create PR if success', async () => { platform.getBranchStatus.mockResolvedValueOnce(BranchStatus.green); - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); config.prCreation = 'status-success'; config.automerge = true; config.schedule = ['before 5am']; @@ -264,7 +259,7 @@ describe('workers/pr/index', () => { }); it('should not create PR if limit is reached', async () => { platform.getBranchStatus.mockResolvedValueOnce(BranchStatus.green); - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); config.prCreation = 'status-success'; config.automerge = true; config.schedule = ['before 5am']; @@ -275,7 +270,7 @@ describe('workers/pr/index', () => { }); it('should create PR if limit is reached but dashboard checked', async () => { platform.getBranchStatus.mockResolvedValueOnce(BranchStatus.green); - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); config.prCreation = 'status-success'; config.automerge = true; config.schedule = ['before 5am']; @@ -315,7 +310,7 @@ describe('workers/pr/index', () => { config.recreateClosed = true; config.rebaseWhen = 'never'; for (const upgrade of config.upgrades) { - upgrade.logJSON = await getChangeLogJSON(upgrade); + upgrade.logJSON = await changelogHelper.getChangeLogJSON(upgrade); } const { pr } = await prWorker.ensurePr(config); expect(pr).toMatchObject({ displayNumber: 'New Pull Request' }); @@ -330,7 +325,7 @@ describe('workers/pr/index', () => { config.schedule = ['before 5am']; config.timezone = 'some timezone'; config.rebaseWhen = 'behind-base-branch'; - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); const { pr } = await prWorker.ensurePr(config); expect(pr).toMatchObject({ displayNumber: 'New Pull Request' }); // FIXME: explicit assert condition @@ -341,7 +336,6 @@ describe('workers/pr/index', () => { }); it('should return null if creating PR fails', async () => { platform.getBranchStatus.mockResolvedValueOnce(BranchStatus.green); - platform.createPr = jest.fn(); platform.createPr.mockImplementationOnce(() => { throw new Error('Validation Failed (422)'); }); @@ -405,6 +399,7 @@ describe('workers/pr/index', () => { config.assignees = ['@foo', 'bar']; config.reviewers = ['foo', '@bar', 'foo@bar.com']; config.filterUnavailableUsers = true; + // optional function is undefined by jest platform.filterUnavailableUsers = jest.fn(); platform.filterUnavailableUsers.mockResolvedValue(['foo']); await prWorker.ensurePr(config); @@ -523,7 +518,7 @@ describe('workers/pr/index', () => { config.semanticCommitScope = null; config.automerge = true; config.schedule = ['before 5am']; - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); const { pr } = await prWorker.ensurePr(config); expect(platform.updatePr.mock.calls).toMatchSnapshot(); expect(platform.updatePr).toHaveBeenCalledTimes(0); @@ -537,7 +532,7 @@ describe('workers/pr/index', () => { config.semanticCommitScope = null; config.automerge = true; config.schedule = ['before 5am']; - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); const { pr } = await prWorker.ensurePr(config); expect(platform.updatePr).toHaveBeenCalledTimes(0); expect(pr).toMatchObject(modifiedPr); @@ -546,7 +541,7 @@ describe('workers/pr/index', () => { config.newValue = '1.2.0'; config.automerge = true; config.schedule = ['before 5am']; - config.logJSON = await getChangeLogJSON(config); + config.logJSON = await changelogHelper.getChangeLogJSON(config); platform.getBranchPr.mockResolvedValueOnce(existingPr); const { pr } = await prWorker.ensurePr(config); // FIXME: explicit assert condition @@ -614,9 +609,8 @@ describe('workers/pr/index', () => { platform.getBranchStatus.mockResolvedValueOnce(BranchStatus.green); config.prCreation = 'status-success'; config.privateRepo = false; - config.logJSON = await getChangeLogJSON(config); - config.logJSON.project.gitlab = 'someproject'; - delete config.logJSON.project.github; + config.logJSON = await changelogHelper.getChangeLogJSON(config); + config.logJSON.project.repository = 'someproject'; const { pr } = await prWorker.ensurePr(config); expect(pr).toMatchObject({ displayNumber: 'New Pull Request' }); expect(platform.createPr.mock.calls[0]).toMatchSnapshot(); diff --git a/lib/workers/pr/index.ts b/lib/workers/pr/index.ts index b3cf078daef997..f6f317cfdde3d3 100644 --- a/lib/workers/pr/index.ts +++ b/lib/workers/pr/index.ts @@ -278,9 +278,7 @@ export async function ensurePr( if (logJSON) { if (typeof logJSON.error === 'undefined') { if (logJSON.project) { - upgrade.repoName = logJSON.project.github - ? logJSON.project.github - : logJSON.project.gitlab; + upgrade.repoName = logJSON.project.repository; } upgrade.hasReleaseNotes = logJSON.hasReleaseNotes; upgrade.releases = []; diff --git a/lib/workers/repository/extract/__snapshots__/manager-files.spec.ts.snap b/lib/workers/repository/extract/__snapshots__/manager-files.spec.ts.snap index 631dbbcf5ccb41..281f2250844f50 100644 --- a/lib/workers/repository/extract/__snapshots__/manager-files.spec.ts.snap +++ b/lib/workers/repository/extract/__snapshots__/manager-files.spec.ts.snap @@ -18,6 +18,7 @@ Array [ "lernaPackages": undefined, "managerData": Object { "lernaJsonFile": undefined, + "yarnZeroInstall": false, }, "npmLock": undefined, "npmrc": undefined, diff --git a/lib/workers/repository/stats.ts b/lib/workers/repository/stats.ts index 3d15ec92fd74e4..86995200d99c61 100644 --- a/lib/workers/repository/stats.ts +++ b/lib/workers/repository/stats.ts @@ -30,25 +30,26 @@ export function printRequestStats(): void { requestHosts[hostname].push(httpRequest); } logger.trace({ allRequests, requestHosts }, 'full stats'); - const hostStats: string[] = []; + type HostStats = { + requestCount: number; + requestAvgMs: number; + queueAvgMs: number; + }; + const hostStats: Record = {}; let totalRequests = 0; for (const [hostname, requests] of Object.entries(requestHosts)) { - const hostRequests = requests.length; - totalRequests += hostRequests; + const requestCount = requests.length; + totalRequests += requestCount; const requestSum = requests .map(({ duration }) => duration) .reduce((a, b) => a + b, 0); - const requestAvg = Math.round(requestSum / hostRequests); + const requestAvgMs = Math.round(requestSum / requestCount); const queueSum = requests .map(({ queueDuration }) => queueDuration) .reduce((a, b) => a + b, 0); - const queueAvg = Math.round(queueSum / hostRequests); - const requestCount = - `${hostRequests} ` + (hostRequests > 1 ? 'requests' : 'request'); - hostStats.push( - `${hostname}, ${requestCount}, ${requestAvg}ms request average, ${queueAvg}ms queue average` - ); + const queueAvgMs = Math.round(queueSum / requestCount); + hostStats[hostname] = { requestCount, requestAvgMs, queueAvgMs }; } logger.debug({ hostStats, totalRequests }, 'http statistics'); } diff --git a/package.json b/package.json index 077749cab91310..4fa965180f9ec6 100644 --- a/package.json +++ b/package.json @@ -235,8 +235,8 @@ "@types/traverse": "0.6.32", "@types/url-join": "4.0.1", "@types/xmldoc": "1.1.6", - "@typescript-eslint/eslint-plugin": "4.29.2", - "@typescript-eslint/parser": "4.29.2", + "@typescript-eslint/eslint-plugin": "4.29.3", + "@typescript-eslint/parser": "4.29.3", "conventional-changelog-conventionalcommits": "4.6.0", "cross-env": "7.0.3", "emojibase-data": "6.2.0", @@ -244,12 +244,12 @@ "eslint-config-airbnb-typescript": "12.3.1", "eslint-config-prettier": "8.3.0", "eslint-formatter-gha": "1.2.0", - "eslint-plugin-import": "2.24.1", + "eslint-plugin-import": "2.24.2", "eslint-plugin-jest": "24.4.0", "eslint-plugin-promise": "5.1.0", "glob": "7.1.7", "graphql": "15.5.1", - "husky": "7.0.1", + "husky": "7.0.2", "jest": "27.0.6", "jest-extended": "0.11.5", "jest-github-actions-reporter": "1.0.3", @@ -258,18 +258,18 @@ "jest-silent-reporter": "0.5.0", "markdownlint-cli2": "0.2.0", "mockdate": "3.0.5", - "nock": "13.1.2", + "nock": "13.1.3", "npm-run-all": "4.1.5", "prettier": "2.3.2", "pretty-quick": "3.1.1", "rimraf": "3.0.2", - "semantic-release": "17.4.5", + "semantic-release": "17.4.6", "shelljs": "0.8.4", "strip-ansi": "6.0.0", "tmp-promise": "3.0.2", "ts-jest": "27.0.5", "ts-node": "10.2.1", - "type-fest": "2.0.0", + "type-fest": "2.1.0", "typescript": "4.3.5", "unified": "9.2.2" }, diff --git a/yarn.lock b/yarn.lock index 6cbcd7ee123aa5..b57d76e64e74c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2103,32 +2103,20 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.2.tgz#f54dc0a32b8f61c6024ab8755da05363b733838d" - integrity sha512-x4EMgn4BTfVd9+Z+r+6rmWxoAzBaapt4QFqE+d8L8sUtYZYLDTK6VG/y/SMMWA5t1/BVU5Kf+20rX4PtWzUYZg== +"@typescript-eslint/eslint-plugin@4.29.3": + version "4.29.3" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.29.3.tgz#95cb8029a8bd8bd9c7f4ab95074a7cb2115adefa" + integrity sha512-tBgfA3K/3TsZY46ROGvoRxQr1wBkclbVqRQep97MjVHJzcRBURRY3sNFqLk0/Xr//BY5hM9H2p/kp+6qim85SA== dependencies: - "@typescript-eslint/experimental-utils" "4.29.2" - "@typescript-eslint/scope-manager" "4.29.2" + "@typescript-eslint/experimental-utils" "4.29.3" + "@typescript-eslint/scope-manager" "4.29.3" debug "^4.3.1" functional-red-black-tree "^1.0.1" regexpp "^3.1.0" semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/experimental-utils@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.29.2.tgz#5f67fb5c5757ef2cb3be64817468ba35c9d4e3b7" - integrity sha512-P6mn4pqObhftBBPAv4GQtEK7Yos1fz/MlpT7+YjH9fTxZcALbiiPKuSIfYP/j13CeOjfq8/fr9Thr2glM9ub7A== - dependencies: - "@types/json-schema" "^7.0.7" - "@typescript-eslint/scope-manager" "4.29.2" - "@typescript-eslint/types" "4.29.2" - "@typescript-eslint/typescript-estree" "4.29.2" - eslint-scope "^5.1.1" - eslint-utils "^3.0.0" - -"@typescript-eslint/experimental-utils@^4.0.1": +"@typescript-eslint/experimental-utils@4.29.3", "@typescript-eslint/experimental-utils@^4.0.1": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/experimental-utils/-/experimental-utils-4.29.3.tgz#52e437a689ccdef73e83c5106b34240a706f15e1" integrity sha512-ffIvbytTVWz+3keg+Sy94FG1QeOvmV9dP2YSdLFHw/ieLXWCa3U1TYu8IRCOpMv2/SPS8XqhM1+ou1YHsdzKrg== @@ -2140,17 +2128,7 @@ eslint-scope "^5.1.1" eslint-utils "^3.0.0" -"@typescript-eslint/parser@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.29.2.tgz#1c7744f4c27aeb74610c955d3dce9250e95c370a" - integrity sha512-WQ6BPf+lNuwteUuyk1jD/aHKqMQ9jrdCn7Gxt9vvBnzbpj7aWEf+aZsJ1zvTjx5zFxGCt000lsbD9tQPEL8u6g== - dependencies: - "@typescript-eslint/scope-manager" "4.29.2" - "@typescript-eslint/types" "4.29.2" - "@typescript-eslint/typescript-estree" "4.29.2" - debug "^4.3.1" - -"@typescript-eslint/parser@^4.4.1": +"@typescript-eslint/parser@4.29.3", "@typescript-eslint/parser@^4.4.1": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-4.29.3.tgz#2ac25535f34c0e98f50c0e6b28c679c2357d45f2" integrity sha512-jrHOV5g2u8ROghmspKoW7pN8T/qUzk0+DITun0MELptvngtMrwUJ1tv5zMI04CYVEUsSrN4jV7AKSv+I0y0EfQ== @@ -2160,14 +2138,6 @@ "@typescript-eslint/typescript-estree" "4.29.3" debug "^4.3.1" -"@typescript-eslint/scope-manager@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.29.2.tgz#442b0f029d981fa402942715b1718ac7fcd5aa1b" - integrity sha512-mfHmvlQxmfkU8D55CkZO2sQOueTxLqGvzV+mG6S/6fIunDiD2ouwsAoiYCZYDDK73QCibYjIZmGhpvKwAB5BOA== - dependencies: - "@typescript-eslint/types" "4.29.2" - "@typescript-eslint/visitor-keys" "4.29.2" - "@typescript-eslint/scope-manager@4.29.3": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-4.29.3.tgz#497dec66f3a22e459f6e306cf14021e40ec86e19" @@ -2176,29 +2146,11 @@ "@typescript-eslint/types" "4.29.3" "@typescript-eslint/visitor-keys" "4.29.3" -"@typescript-eslint/types@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.29.2.tgz#fc0489c6b89773f99109fb0aa0aaddff21f52fcd" - integrity sha512-K6ApnEXId+WTGxqnda8z4LhNMa/pZmbTFkDxEBLQAbhLZL50DjeY0VIDCml/0Y3FlcbqXZrABqrcKxq+n0LwzQ== - "@typescript-eslint/types@4.29.3": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-4.29.3.tgz#d7980c49aef643d0af8954c9f14f656b7fd16017" integrity sha512-s1eV1lKNgoIYLAl1JUba8NhULmf+jOmmeFO1G5MN/RBCyyzg4TIOfIOICVNC06lor+Xmy4FypIIhFiJXOknhIg== -"@typescript-eslint/typescript-estree@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.29.2.tgz#a0ea8b98b274adbb2577100ba545ddf8bf7dc219" - integrity sha512-TJ0/hEnYxapYn9SGn3dCnETO0r+MjaxtlWZ2xU+EvytF0g4CqTpZL48SqSNn2hXsPolnewF30pdzR9a5Lj3DNg== - dependencies: - "@typescript-eslint/types" "4.29.2" - "@typescript-eslint/visitor-keys" "4.29.2" - debug "^4.3.1" - globby "^11.0.3" - is-glob "^4.0.1" - semver "^7.3.5" - tsutils "^3.21.0" - "@typescript-eslint/typescript-estree@4.29.3": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-4.29.3.tgz#1bafad610015c4ded35c85a70b6222faad598b40" @@ -2212,14 +2164,6 @@ semver "^7.3.5" tsutils "^3.21.0" -"@typescript-eslint/visitor-keys@4.29.2": - version "4.29.2" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.29.2.tgz#d2da7341f3519486f50655159f4e5ecdcb2cd1df" - integrity sha512-bDgJLQ86oWHJoZ1ai4TZdgXzJxsea3Ee9u9wsTAvjChdj2WLcVsgWYAPeY7RQMn16tKrlQaBnpKv7KBfs4EQag== - dependencies: - "@typescript-eslint/types" "4.29.2" - eslint-visitor-keys "^2.0.0" - "@typescript-eslint/visitor-keys@4.29.3": version "4.29.3" resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-4.29.3.tgz#c691760a00bd86bf8320d2a90a93d86d322f1abf" @@ -4045,10 +3989,10 @@ eslint-module-utils@^2.6.2: debug "^3.2.7" pkg-dir "^2.0.0" -eslint-plugin-import@2.24.1: - version "2.24.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.24.1.tgz#64aba8b567a1ba9921d5465586e86c491b8e2135" - integrity sha512-KSFWhNxPH8OGJwpRJJs+Z7I0a13E2iFQZJIvSnCu6KUs4qmgAm3xN9GYBCSoiGWmwA7gERZPXqYQjcoCROnYhQ== +eslint-plugin-import@2.24.2: + version "2.24.2" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.24.2.tgz#2c8cd2e341f3885918ee27d18479910ade7bb4da" + integrity sha512-hNVtyhiEtZmpsabL4neEj+6M5DCLgpYyG9nzJY8lZQeQXEn5UPW1DpUdsMHMXsq98dbNm7nt1w9ZMSVpfJdi8Q== dependencies: array-includes "^3.1.3" array.prototype.flat "^1.2.4" @@ -4064,7 +4008,7 @@ eslint-plugin-import@2.24.1: pkg-up "^2.0.0" read-pkg-up "^3.0.0" resolve "^1.20.0" - tsconfig-paths "^3.10.1" + tsconfig-paths "^3.11.0" eslint-plugin-jest@24.4.0: version "24.4.0" @@ -4999,10 +4943,10 @@ humanize-ms@^1.2.1: dependencies: ms "^2.0.0" -husky@7.0.1: - version "7.0.1" - resolved "https://registry.yarnpkg.com/husky/-/husky-7.0.1.tgz#579f4180b5da4520263e8713cc832942b48e1f1c" - integrity sha512-gceRaITVZ+cJH9sNHqx5tFwbzlLCVxtVZcusME8JYQ8Edy5mpGDOqD8QBCdMhpyo9a+JXddnujQ4rpY2Ff9SJA== +husky@7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/husky/-/husky-7.0.2.tgz#21900da0f30199acca43a46c043c4ad84ae88dff" + integrity sha512-8yKEWNX4z2YsofXAMT7KvA1g8p+GxtB1ffV8XtpAEGuXNAbCV5wdNKH+qTpw8SM9fh4aMPDR+yQuKfgnreyZlg== iconv-lite@0.4.24: version "0.4.24" @@ -6668,10 +6612,10 @@ marked-terminal@^4.1.1: node-emoji "^1.10.0" supports-hyperlinks "^2.1.0" -marked@^3.0.0: - version "3.0.2" - resolved "https://registry.yarnpkg.com/marked/-/marked-3.0.2.tgz#60ce97d6aec34dd882ab4bb4df82494666854e17" - integrity sha512-TMJQQ79Z0e3rJYazY0tIoMsFzteUGw9fB3FD+gzuIT3zLuG9L9ckIvUfF51apdJkcqc208jJN2KbtPbOvXtbjA== +marked@^2.0.0: + version "2.1.3" + resolved "https://registry.yarnpkg.com/marked/-/marked-2.1.3.tgz#bd017cef6431724fd4b27e0657f5ceb14bff3753" + integrity sha512-/Q+7MGzaETqifOMWYEA7HVMaZb4XbcRfaOzcSsHZEith83KGlvaSG33u0SKu89Mj5h+T8V2hM+8O45Qc5XTgwA== marshal@0.5.2: version "0.5.2" @@ -7089,10 +7033,10 @@ nice-try@^1.0.4: resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.5.tgz#a3378a7696ce7d223e88fc9b764bd7ef1089e366" integrity sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ== -nock@13.1.2: - version "13.1.2" - resolved "https://registry.yarnpkg.com/nock/-/nock-13.1.2.tgz#eedcef89e1c1e8db4e1b3cdcfef91f453f6d32b5" - integrity sha512-BDjokoeGZnBghmvwCcDJ1yM5TDRMRAJfGi1xIzX5rKTlifbyx1oRpAVl3aNhEA3kGbUSEPD7gBLmwVdnQibrIA== +nock@13.1.3: + version "13.1.3" + resolved "https://registry.yarnpkg.com/nock/-/nock-13.1.3.tgz#110b005965654a8ffb798e87bad18b467bff15f9" + integrity sha512-YKj0rKQWMGiiIO+Y65Ut8OEgYM3PplLU2+GAhnPmqZdBd6z5IskgdBqWmjzA6lH3RF0S2a3wiAlrMOF5Iv2Jeg== dependencies: debug "^4.1.0" json-stringify-safe "^5.0.1" @@ -8519,10 +8463,10 @@ saxes@^5.0.1: dependencies: xmlchars "^2.2.0" -semantic-release@17.4.5: - version "17.4.5" - resolved "https://registry.yarnpkg.com/semantic-release/-/semantic-release-17.4.5.tgz#b636065f3fdc8392e2fdfeac0e99d9c3805f8a3a" - integrity sha512-s9JqU46nK1TQla37+kA7aSoNWDCkQq9/T5K2QX/z7bfI+zW3o1Pg/2nlZuZffnnROnt5sT0ysE49yhedINTtTw== +semantic-release@17.4.6: + version "17.4.6" + resolved "https://registry.yarnpkg.com/semantic-release/-/semantic-release-17.4.6.tgz#a9e53da3d43ac07250c55b6939857587be2ecaa2" + integrity sha512-0B1OGkW8gzQjLbj3H5G4fJyy73HAWsoDnzWYyfNJHyyyLi1acQd+id4O7U+12wXTk9G83pXCgSFj9oYTvocFMA== dependencies: "@semantic-release/commit-analyzer" "^8.0.0" "@semantic-release/error" "^2.2.0" @@ -8541,7 +8485,7 @@ semantic-release@17.4.5: hook-std "^2.0.0" hosted-git-info "^4.0.0" lodash "^4.17.21" - marked "^3.0.0" + marked "^2.0.0" marked-terminal "^4.1.1" micromatch "^4.0.2" p-each-series "^2.1.0" @@ -9418,7 +9362,7 @@ ts-node@10.2.1: make-error "^1.1.1" yn "3.1.1" -tsconfig-paths@^3.10.1: +tsconfig-paths@^3.11.0: version "3.11.0" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.11.0.tgz#954c1fe973da6339c78e06b03ce2e48810b65f36" integrity sha512-7ecdYDnIdmv639mmDwslG6KQg1Z9STTz1j7Gcz0xa+nshh/gKDAHcPxRbWOsA3SPp0tXP2leTcY9Kw+NAkfZzA== @@ -9486,10 +9430,10 @@ type-detect@4.0.8: resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g== -type-fest@2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.0.0.tgz#e9daf5615e89f6d430f34117f88f4ee2cd5a2725" - integrity sha512-BoEUnckjP9oiudy3KxlGdudtBAdJQ74Wp7dYwVPkUzBn+cVHOsBXh2zD2jLyqgbuJ1KMNriczZCI7lTBA94dFg== +type-fest@2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-2.1.0.tgz#1f8b20ff51519f3b01b3188d50dea9f9ebfbf1b8" + integrity sha512-2wHUmKDy5wNLmebekbHx/zE9ElYAKOmz34psTLG7OwyEJHaIUr6jnaCd55EvgrawAvliwbwgbyH1LkxIfWFyNg== type-fest@^0.13.1: version "0.13.1"