Skip to content

Commit

Permalink
refactor(composer): Simplify Packagist lookups (#19808)
Browse files Browse the repository at this point in the history
  • Loading branch information
zharinov committed Jan 12, 2023
1 parent 84eed32 commit 4b2f376
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 33 deletions.
35 changes: 30 additions & 5 deletions lib/modules/datasource/packagist/index.spec.ts
Expand Up @@ -82,7 +82,12 @@ describe('modules/datasource/packagist/index', () => {
.scope('https://composer.renovatebot.com')
.get('/packages.json')
.replyWithError({ code: 'ETIMEDOUT' });
httpMock.scope(baseUrl).get('/p2/vendor/package-name2.json').reply(200);
httpMock
.scope(baseUrl)
.get('/p2/vendor/package-name2.json')
.reply(200)
.get('/p2/vendor/package-name2~dev.json')
.reply(200);
const res = await getPkgReleases({
...config,
datasource,
Expand All @@ -97,7 +102,12 @@ describe('modules/datasource/packagist/index', () => {
.scope('https://composer.renovatebot.com')
.get('/packages.json')
.reply(403);
httpMock.scope(baseUrl).get('/p2/vendor/package-name.json').reply(200);
httpMock
.scope(baseUrl)
.get('/p2/vendor/package-name.json')
.reply(200)
.get('/p2/vendor/package-name~dev.json')
.reply(200);
const res = await getPkgReleases({
...config,
datasource,
Expand All @@ -112,7 +122,12 @@ describe('modules/datasource/packagist/index', () => {
.scope('https://composer.renovatebot.com')
.get('/packages.json')
.reply(404);
httpMock.scope(baseUrl).get('/p2/drewm/mailchimp-api.json').reply(200);
httpMock
.scope(baseUrl)
.get('/p2/drewm/mailchimp-api.json')
.reply(200)
.get('/p2/drewm/mailchimp-api~dev.json')
.reply(200);
const res = await getPkgReleases({
...config,
datasource,
Expand Down Expand Up @@ -266,7 +281,12 @@ describe('modules/datasource/packagist/index', () => {
'/p/providers-2018-09$14346045d7a7261cb3a12a6b7a1a7c4151982530347b115e5e277d879cad1942.json'
)
.reply(200, fileJson);
httpMock.scope(baseUrl).get('/p2/some/other.json').reply(200, beytJson);
httpMock
.scope(baseUrl)
.get('/p2/some/other.json')
.reply(200, beytJson)
.get('/p2/some/other~dev.json')
.reply(200, beytJson);
const res = await getPkgReleases({
...config,
datasource,
Expand Down Expand Up @@ -357,7 +377,12 @@ describe('modules/datasource/packagist/index', () => {
.scope('https://composer.renovatebot.com')
.get('/packages.json')
.reply(200, packagesJson);
httpMock.scope(baseUrl).get('/p2/some/other.json').reply(200, beytJson);
httpMock
.scope(baseUrl)
.get('/p2/some/other.json')
.reply(200, beytJson)
.get('/p2/some/other~dev.json')
.reply(200, beytJson);
const res = await getPkgReleases({
...config,
datasource,
Expand Down
40 changes: 12 additions & 28 deletions lib/modules/datasource/packagist/index.ts
@@ -1,7 +1,6 @@
import URL from 'url';
import { logger } from '../../../logger';
import { ExternalHostError } from '../../../types/errors/external-host-error';
import * as packageCache from '../../../util/cache/package';
import { cache } from '../../../util/cache/package/decorator';
import * as hostRules from '../../../util/host-rules';
import type { HttpOptions } from '../../../util/http/types';
Expand All @@ -11,6 +10,7 @@ import { ensureTrailingSlash, joinUrlParts } from '../../../util/url';
import * as composerVersioning from '../../versioning/composer';
import { Datasource } from '../datasource';
import type { GetReleasesConfig, ReleaseResult } from '../types';
import * as schema from './schema';
import type {
AllPackages,
PackageMeta,
Expand Down Expand Up @@ -211,35 +211,19 @@ export class PackagistDatasource extends Datasource {
return allPackages;
}

@cache({
namespace: `datasource-${PackagistDatasource.id}-org`,
key: (regUrl: string) => regUrl,
ttlMinutes: 10,
})
async packagistOrgLookup(name: string): Promise<ReleaseResult | null> {
const cacheNamespace = 'datasource-packagist-org';
const cachedResult = await packageCache.get<ReleaseResult>(
cacheNamespace,
name
);
// istanbul ignore if
if (cachedResult) {
return cachedResult;
}
let dep: ReleaseResult | null = null;
const regUrl = 'https://packagist.org';
const pkgUrl = [
joinUrlParts(regUrl, `/p2/${name}.json`),
joinUrlParts(regUrl, `/p2/${name}~dev.json`),
];
// TODO: fix types (#9610)
let res = (await this.http.getJson<any>(pkgUrl[0])).body.packages[name];
res = [
...res,
...(await this.http.getJson<any>(pkgUrl[1])).body.packages[name],
];
if (res) {
dep = PackagistDatasource.extractDepReleases(res);
logger.trace({ dep }, 'dep');
}
const cacheMinutes = 10;
await packageCache.set(cacheNamespace, name, dep, cacheMinutes);
return dep;
const pkgUrl = joinUrlParts(regUrl, `/p2/${name}.json`);
const devUrl = joinUrlParts(regUrl, `/p2/${name}~dev.json`);
const results = await p.map([pkgUrl, devUrl], (url) =>
this.http.getJson(url).then(({ body }) => body)
);
return schema.ComposerV2ReleaseResult.parse(results);
}

private async packageLookup(
Expand Down
67 changes: 67 additions & 0 deletions lib/modules/datasource/packagist/schema.ts
@@ -0,0 +1,67 @@
import { z } from 'zod';
import { api as versioning } from '../../versioning/composer';
import type { Release, ReleaseResult } from '../types';

export const PackageName = z
.string()
.refine((s) => s.split('/').length === 2, 'Invalid package name');

export const ComposerV2Release = z.object({
version: z.string(),
homepage: z.optional(z.string().url()),
source: z.optional(
z.object({
url: z.string().url(),
})
),
time: z.string().datetime({ offset: true }),
});

export const ComposerV2PackageResponse = z.object({
packages: z.record(PackageName, z.array(ComposerV2Release)),
});

export const ComposerV2ReleaseResult = z
.array(ComposerV2PackageResponse)
.transform((responses): ReleaseResult => {
const releases: Release[] = [];
let maxVersion: string | undefined;
let homepage: string | undefined = undefined;
let sourceUrl: string | undefined = undefined;

for (const response of responses) {
for (const responsePackage of Object.values(response.packages)) {
for (const composerV2Release of responsePackage) {
const { version, time: releaseTimestamp } = composerV2Release;
const dep: Release = {
version: version.replace(/^v/, ''),
gitRef: version,
releaseTimestamp,
};
releases.push(dep);

if (!versioning.isValid(version)) {
continue;
}

if (!maxVersion || versioning.isGreaterThan(version, maxVersion)) {
maxVersion = version;
homepage = composerV2Release.homepage;
sourceUrl = composerV2Release.source?.url;
}
}
}
}

const result: ReleaseResult = { releases };

if (homepage) {
result.homepage = homepage;
}

if (sourceUrl) {
result.sourceUrl = sourceUrl;
}

return result;
});

0 comments on commit 4b2f376

Please sign in to comment.