Skip to content

Commit

Permalink
refactor(go): Change iteration over major versions (#22900)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zharinov
zharinov committed Jun 21, 2023
1 parent 6de815e commit 192fe84
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 56 deletions.
53 changes: 45 additions & 8 deletions lib/modules/datasource/go/releases-goproxy.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { codeBlock } from 'common-tags';
import { Fixtures } from '../../../../test/fixtures';
import * as httpMock from '../../../../test/http-mock';
import { GithubReleasesDatasource } from '../github-releases';
Expand Down Expand Up @@ -340,7 +341,10 @@ describe('modules/datasource/go/releases-goproxy', () => {
.get('/@v/list')
.reply(
200,
['v1.0.0 2018-08-13T15:31:12Z', 'v1.0.1', ' \n'].join('\n')
codeBlock`
v1.0.0 2018-08-13T15:31:12Z
v1.0.1
`
)
.get('/@v/v1.0.1.info')
.reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' })
Expand All @@ -366,13 +370,21 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.replyWithError('unknown')
.get('/@v/v1.0.1.info')
.reply(410)
.get('/v2/@v/list')
.reply(200);
.reply(200)
.get('/v3/@v/list')
.reply(404);

const res = await datasource.getReleases({
packageName: 'github.com/google/btree',
Expand All @@ -395,7 +407,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
Expand Down Expand Up @@ -436,7 +454,13 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
Expand Down Expand Up @@ -580,13 +604,24 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200, 'v1.0.0\nv1.0.1\n')
.reply(
200,
codeBlock`
v1.0.0
v1.0.1
`
)
.get('/@v/v1.0.0.info')
.reply(200, { Version: 'v1.0.0', Time: '2018-08-13T15:31:12Z' })
.get('/@v/v1.0.1.info')
.reply(200, { Version: 'v1.0.1', Time: '2019-10-16T16:15:28Z' })
.get('/v2/@v/list')
.reply(200, 'v2.0.0\n')
.reply(
200,
codeBlock`
v2.0.0
`
)
.get('/v2/@v/v2.0.0.info')
.reply(200, { Version: 'v2.0.0', Time: '2020-10-16T16:15:28Z' })
.get('/v3/@v/list')
Expand Down Expand Up @@ -679,7 +714,9 @@ describe('modules/datasource/go/releases-goproxy', () => {
httpMock
.scope(`${baseUrl}/github.com/google/btree`)
.get('/@v/list')
.reply(200);
.reply(200)
.get('/v2/@v/list')
.reply(404);

const res = await datasource.getReleases({
packageName: 'github.com/google/btree',
Expand Down
97 changes: 49 additions & 48 deletions lib/modules/datasource/go/releases-goproxy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { DateTime } from 'luxon';
import moo from 'moo';
import { logger } from '../../../logger';
import { cache } from '../../../util/cache/package/decorator';
import { HttpError } from '../../../util/http';
import * as p from '../../../util/promises';
import { newlineRegex, regEx } from '../../../util/regex';
import { Datasource } from '../datasource';
Expand All @@ -14,7 +15,7 @@ import type { GoproxyItem, VersionInfo } from './types';

const parsedGoproxy: Record<string, GoproxyItem[]> = {};

const modRegex = regEx(`^(?<baseMod>.*?)(?:[./]v(?<majorVersion>\\d+))?$`);
const modRegex = regEx(/^(?<baseMod>.*?)(?:[./]v(?<majorVersion>\d+))?$/);

export class GoProxyDatasource extends Datasource {
static readonly id = 'go-proxy';
Expand Down Expand Up @@ -47,12 +48,6 @@ export class GoProxyDatasource extends Datasource {
return result;
}

const isGopkgin = packageName.startsWith('gopkg.in/');
const majorSuffixSeparator = isGopkgin ? '.' : '/';
const modParts = packageName.match(modRegex);
const baseMod = modParts?.groups?.baseMod ?? packageName;
const currentMajor = parseInt(modParts?.groups?.majorVersion ?? '0');

for (const { url, fallback } of proxyList) {
try {
if (url === 'off') {
Expand All @@ -62,43 +57,20 @@ export class GoProxyDatasource extends Datasource {
break;
}

const releases: Release[] = [];
for (let i = 0; ; i++) {
try {
const major = currentMajor + i;
let mod: string;
if (major < 2 && !isGopkgin) {
mod = baseMod;
i++; // v0 and v1 are the same module
} else {
mod = `${baseMod}${majorSuffixSeparator}v${major}`;
}
const result = await this.getVersionsWithInfo(url, mod);
if (result.length === 0) {
break;
}
releases.push(...result);
} catch (err) {
const statusCode = err?.response?.statusCode;
if (i > 0 && statusCode === 404) {
break;
}
throw err;
}
}

if (releases.length) {
const res = await this.getVersionsWithInfo(url, packageName);
if (res.releases.length) {
result = res;
try {
const datasource = await BaseGoDatasource.getDatasource(
packageName
);
const sourceUrl = getSourceUrl(datasource) ?? null;
result = { releases, sourceUrl };
const sourceUrl = getSourceUrl(datasource);
if (sourceUrl) {
result.sourceUrl = sourceUrl;
}
} catch (err) {
logger.trace({ err }, `Can't get datasource for ${packageName}`);
result = { releases };
}

break;
}
} catch (err) {
Expand Down Expand Up @@ -263,23 +235,52 @@ export class GoProxyDatasource extends Datasource {
async getVersionsWithInfo(
baseUrl: string,
packageName: string
): Promise<Release[]> {
const releasesIndex = await this.listVersions(baseUrl, packageName);
const releases = await p.map(releasesIndex, async (versionInfo) => {
const { version, releaseTimestamp } = versionInfo;
): Promise<ReleaseResult> {
const isGopkgin = packageName.startsWith('gopkg.in/');
const majorSuffixSeparator = isGopkgin ? '.' : '/';
const modParts = packageName.match(modRegex)?.groups;
const baseMod = modParts?.baseMod ?? /* istanbul ignore next */ packageName;
const packageMajor = parseInt(modParts?.majorVersion ?? '0');

if (releaseTimestamp) {
return { version, releaseTimestamp };
const result: ReleaseResult = { releases: [] };
for (let major = packageMajor; ; major += 1) {
let pkg = `${baseMod}${majorSuffixSeparator}v${major}`;
if (!isGopkgin && major < 2) {
pkg = baseMod;
major += 1; // v0 and v1 are the same module
}

try {
return await this.versionInfo(baseUrl, packageName, version);
const res = await this.listVersions(baseUrl, pkg);
const releases = await p.map(res, async (versionInfo) => {
const { version, releaseTimestamp } = versionInfo;

if (releaseTimestamp) {
return { version, releaseTimestamp };
}

try {
return await this.versionInfo(baseUrl, pkg, version);
} catch (err) {
logger.trace({ err }, `Can't obtain data from ${baseUrl}`);
return { version };
}
});
result.releases.push(...releases);
} catch (err) {
logger.trace({ err }, `Can't obtain data from ${baseUrl}`);
return { version };
if (
err instanceof HttpError &&
err.response?.statusCode === 404 &&
major !== packageMajor
) {
break;
}

throw err;
}
});
return releases;
}

return result;
}

static getCacheKey({ packageName }: GetReleasesConfig): string {
Expand Down

0 comments on commit 192fe84

Please sign in to comment.