From bc824eb4e6c7b03d61443194d4e0acf3dadfb15b Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Thu, 1 Feb 2024 08:03:57 +0100 Subject: [PATCH 1/6] fix(datasource/go): private repositories on azure This will fix a bug that occurs with private go packages with a major version greater than 1 on Azure DevOps. The problem is that Azure DevOps has support for go-get metadata but it will only work for modules that are included without a version suffix attached, like dev.azure.com///_git/.git/v2 Once such a suffix is part of the URL the http request to fetch the metadata info will result in 404 Not Found error. This change will now fetch the releases for the module as git tags from the repository. The Azure DevOps documentation https://learn.microsoft.com/en-us/azure/devops/repos/git/go-get describes two ways to name go packages: dev.azure.com///_git/.git and dev.azure.com///.git Both possible names are changed to https://dev.azure.com///_git/ to have a valid Azure DevOps https git URL. --- lib/modules/datasource/go/base.spec.ts | 33 +++++++------------------- lib/modules/datasource/go/base.ts | 18 ++++++++++++++ 2 files changed, 26 insertions(+), 25 deletions(-) diff --git a/lib/modules/datasource/go/base.spec.ts b/lib/modules/datasource/go/base.spec.ts index 7bfbe623ae5589..80df3ee091a7f8 100644 --- a/lib/modules/datasource/go/base.spec.ts +++ b/lib/modules/datasource/go/base.spec.ts @@ -16,12 +16,14 @@ const hostRules = mocked(_hostRules); describe('modules/datasource/go/base', () => { describe('simple cases', () => { it.each` - module | datasource | packageName - ${'gopkg.in/foo'} | ${'github-tags'} | ${'go-foo/foo'} - ${'gopkg.in/foo/bar'} | ${'github-tags'} | ${'foo/bar'} - ${'github.com/foo/bar'} | ${'github-tags'} | ${'foo/bar'} - ${'bitbucket.org/foo/bar'} | ${'bitbucket-tags'} | ${'foo/bar'} - ${'code.cloudfoundry.org/lager'} | ${'github-tags'} | ${'cloudfoundry/lager'} + module | datasource | packageName + ${'gopkg.in/foo'} | ${'github-tags'} | ${'go-foo/foo'} + ${'gopkg.in/foo/bar'} | ${'github-tags'} | ${'foo/bar'} + ${'github.com/foo/bar'} | ${'github-tags'} | ${'foo/bar'} + ${'bitbucket.org/foo/bar'} | ${'bitbucket-tags'} | ${'foo/bar'} + ${'code.cloudfoundry.org/lager'} | ${'github-tags'} | ${'cloudfoundry/lager'} + ${'dev.azure.com/foo/bar/_git/baz.git'} | ${'git-tags'} | ${'https://dev.azure.com/foo/bar/_git/baz'} + ${'dev.azure.com/foo/bar/baz.git'} | ${'git-tags'} | ${'https://dev.azure.com/foo/bar/_git/baz'} `( '$module -> $datasource: $packageName', async ({ module, datasource, packageName }) => { @@ -345,25 +347,6 @@ describe('modules/datasource/go/base', () => { }); }); - it('handles go-import with azure devops source', async () => { - const meta = - ''; - httpMock - .scope('https://dev.azure.com') - .get('/my-organization/my-project/_git/my-repo.git?go-get=1') - .reply(200, meta); - - const res = await BaseGoDatasource.getDatasource( - 'dev.azure.com/my-organization/my-project/_git/my-repo.git', - ); - - expect(res).toEqual({ - datasource: GitTagsDatasource.id, - packageName: - 'https://dev.azure.com/my-organization/my-project/_git/my-repo', - }); - }); - it('handles uncommon imports', async () => { const meta = ''; diff --git a/lib/modules/datasource/go/base.ts b/lib/modules/datasource/go/base.ts index f109920d3d1a3b..ab95c778944f17 100644 --- a/lib/modules/datasource/go/base.ts +++ b/lib/modules/datasource/go/base.ts @@ -71,6 +71,24 @@ export class BaseGoDatasource { }; } + if (goModule.startsWith('dev.azure.com/')) { + const split = goModule.split('/'); + const packageName = + 'https://dev.azure.com/' + + split[1] + + '/' + + split[2] + + '/_git/' + + (split[3] === '_git' ? split[4] : split[3]).replace( + regEx(/\.git$/), + '', + ); + return { + datasource: GitTagsDatasource.id, + packageName, + }; + } + return await BaseGoDatasource.goGetDatasource(goModule); } From 9529f51ae731359697eb7b8b9b2c225152dfc4b9 Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Fri, 2 Feb 2024 09:08:47 +0100 Subject: [PATCH 2/6] fix: add sanity check for split length --- lib/modules/datasource/go/base.ts | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/lib/modules/datasource/go/base.ts b/lib/modules/datasource/go/base.ts index ab95c778944f17..8ca5cef6bac784 100644 --- a/lib/modules/datasource/go/base.ts +++ b/lib/modules/datasource/go/base.ts @@ -73,20 +73,22 @@ export class BaseGoDatasource { if (goModule.startsWith('dev.azure.com/')) { const split = goModule.split('/'); - const packageName = - 'https://dev.azure.com/' + - split[1] + - '/' + - split[2] + - '/_git/' + - (split[3] === '_git' ? split[4] : split[3]).replace( - regEx(/\.git$/), - '', - ); - return { - datasource: GitTagsDatasource.id, - packageName, - }; + if ((split.length > 4 && split[3] === '_git') || split.length > 3) { + const packageName = + 'https://dev.azure.com/' + + split[1] + + '/' + + split[2] + + '/_git/' + + (split[3] === '_git' ? split[4] : split[3]).replace( + regEx(/\.git$/), + '', + ); + return { + datasource: GitTagsDatasource.id, + packageName, + }; + } } return await BaseGoDatasource.goGetDatasource(goModule); From 9f357705a739757e95a9c7622aab605f858246c9 Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Mon, 26 Feb 2024 10:52:20 +0100 Subject: [PATCH 3/6] feat: adding test for invalid azure devops source --- lib/modules/datasource/go/base.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/modules/datasource/go/base.spec.ts b/lib/modules/datasource/go/base.spec.ts index 80df3ee091a7f8..74474ee248ae65 100644 --- a/lib/modules/datasource/go/base.spec.ts +++ b/lib/modules/datasource/go/base.spec.ts @@ -347,6 +347,19 @@ describe('modules/datasource/go/base', () => { }); }); + it('returns null for invalid azure devops source', async () => { + httpMock + .scope('https://dev.azure.com') + .get('/foo/bar.git?go-get=1') + .reply(200); + + const res = await BaseGoDatasource.getDatasource( + 'dev.azure.com/foo/bar.git', + ); + + expect(res).toBeNull(); + }); + it('handles uncommon imports', async () => { const meta = ''; From 6114d037449af9d09e6c750734f77d0926528943 Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Wed, 28 Feb 2024 08:54:35 +0100 Subject: [PATCH 4/6] fix: make coverage test happy with the recent changes there is no chanche to trigger the azure fall back detection. --- lib/modules/datasource/go/base.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/modules/datasource/go/base.ts b/lib/modules/datasource/go/base.ts index 375987839bfb84..fb4b508ca9e3c3 100644 --- a/lib/modules/datasource/go/base.ts +++ b/lib/modules/datasource/go/base.ts @@ -274,6 +274,7 @@ export class BaseGoDatasource { }; } case 'azure': { + /* istanbul ignore next: can never happen, makes lint happy */ return { datasource: GitTagsDatasource.id, packageName: goImportURL.replace(regEx(/\.git$/), ''), From 9e10b60a9f169cc3162bd7e2300f4693adae55e1 Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Thu, 29 Feb 2024 09:03:58 +0100 Subject: [PATCH 5/6] fix: test go-import with custom azure devops domain --- lib/modules/datasource/go/base.spec.ts | 16 ++++++++++++++++ lib/modules/datasource/go/base.ts | 1 - 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/modules/datasource/go/base.spec.ts b/lib/modules/datasource/go/base.spec.ts index 8bc7b1357f2f0b..2698f34a8ff48e 100644 --- a/lib/modules/datasource/go/base.spec.ts +++ b/lib/modules/datasource/go/base.spec.ts @@ -360,6 +360,22 @@ describe('modules/datasource/go/base', () => { expect(res).toBeNull(); }); + it('handles go-import with azure devops source', async () => { + const meta = + ''; + httpMock + .scope('https://org.visualstudio.com') + .get('/my-project/_git/my-repo.git?go-get=1') + .reply(200, meta); + const res = await BaseGoDatasource.getDatasource( + 'org.visualstudio.com/my-project/_git/my-repo.git', + ); + expect(res).toEqual({ + datasource: GitTagsDatasource.id, + packageName: 'https://org.visualstudio.com/my-project/_git/my-repo', + }); + }); + it('handles uncommon imports', async () => { const meta = ''; diff --git a/lib/modules/datasource/go/base.ts b/lib/modules/datasource/go/base.ts index fb4b508ca9e3c3..375987839bfb84 100644 --- a/lib/modules/datasource/go/base.ts +++ b/lib/modules/datasource/go/base.ts @@ -274,7 +274,6 @@ export class BaseGoDatasource { }; } case 'azure': { - /* istanbul ignore next: can never happen, makes lint happy */ return { datasource: GitTagsDatasource.id, packageName: goImportURL.replace(regEx(/\.git$/), ''), From 428f0f680fe351a85f604b0b45883afdf0f4d52c Mon Sep 17 00:00:00 2001 From: Jan Sperling Date: Thu, 29 Feb 2024 11:00:25 +0100 Subject: [PATCH 6/6] fix: move new test to have less changes --- lib/modules/datasource/go/base.spec.ts | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/modules/datasource/go/base.spec.ts b/lib/modules/datasource/go/base.spec.ts index 2698f34a8ff48e..3c8a08400de554 100644 --- a/lib/modules/datasource/go/base.spec.ts +++ b/lib/modules/datasource/go/base.spec.ts @@ -347,19 +347,6 @@ describe('modules/datasource/go/base', () => { }); }); - it('returns null for invalid azure devops source', async () => { - httpMock - .scope('https://dev.azure.com') - .get('/foo/bar.git?go-get=1') - .reply(200); - - const res = await BaseGoDatasource.getDatasource( - 'dev.azure.com/foo/bar.git', - ); - - expect(res).toBeNull(); - }); - it('handles go-import with azure devops source', async () => { const meta = ''; @@ -376,6 +363,19 @@ describe('modules/datasource/go/base', () => { }); }); + it('returns null for invalid azure devops source', async () => { + httpMock + .scope('https://dev.azure.com') + .get('/foo/bar.git?go-get=1') + .reply(200); + + const res = await BaseGoDatasource.getDatasource( + 'dev.azure.com/foo/bar.git', + ); + + expect(res).toBeNull(); + }); + it('handles uncommon imports', async () => { const meta = '';