From 6008ff7896aa9c3aa578e68ff973468f9c93a143 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 19 May 2023 15:50:40 +0300 Subject: [PATCH 1/5] refactor(npm): Use schema for `PackageSource` parsing --- lib/modules/datasource/npm/get.ts | 64 ++++++++++++++++++------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index f83725b742d1c4..2a7ab6df9c2f88 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -1,6 +1,7 @@ import url from 'node:url'; import is from '@sindresorhus/is'; import { DateTime } from 'luxon'; +import { z } from 'zod'; import { GlobalConfig } from '../../../config/global'; import { HOST_DISABLED } from '../../../constants/error-messages'; import { logger } from '../../../logger'; @@ -13,11 +14,6 @@ import { joinUrlParts } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; import type { CachedReleaseResult, NpmResponse } from './types'; -interface PackageSource { - sourceUrl?: string; - sourceDirectory?: string; -} - const SHORT_REPO_REGEX = regEx( /^((?bitbucket|github|gitlab):)?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)$/ ); @@ -28,27 +24,43 @@ const platformMapping: Record = { gitlab: 'https://gitlab.com/', }; -function getPackageSource(repository: any): PackageSource { - const res: PackageSource = {}; - if (repository) { - if (is.nonEmptyString(repository)) { - const shortMatch = repository.match(SHORT_REPO_REGEX); - if (shortMatch?.groups) { - const { platform = 'github', shortRepo } = shortMatch.groups; - res.sourceUrl = platformMapping[platform] + shortRepo; - } else { - res.sourceUrl = repository; - } - } else if (is.nonEmptyString(repository.url)) { - res.sourceUrl = repository.url; - } - if (is.nonEmptyString(repository.directory)) { - res.sourceDirectory = repository.directory; - } - } - return res; +interface PackageSource { + sourceUrl?: string; + sourceDirectory?: string; } +const PackageSource = z + .union([ + z + .string() + .nonempty() + .transform((repository): PackageSource => { + const shortMatch = repository.match(SHORT_REPO_REGEX); + if (shortMatch?.groups) { + const { platform = 'github', shortRepo } = shortMatch.groups; + return { sourceUrl: platformMapping[platform] + shortRepo }; + } else { + return { sourceUrl: repository }; + } + }), + z + .object({ + url: z.string().nonempty().optional(), + directory: z.string().nonempty().optional(), + }) + .transform(({ url, directory }) => { + const res: PackageSource = {}; + if (url) { + res.sourceUrl = url; + } + if (directory) { + res.sourceDirectory = directory; + } + return res; + }), + ]) + .catch({}); + export async function getDependency( http: Http, registryUrl: string, @@ -124,7 +136,7 @@ export async function getDependency( res.repository ??= latestVersion?.repository; res.homepage ??= latestVersion?.homepage; - const { sourceUrl, sourceDirectory } = getPackageSource(res.repository); + const { sourceUrl, sourceDirectory } = PackageSource.parse(res.repository); // Simplify response before caching and returning const dep: ReleaseResult = { @@ -152,7 +164,7 @@ export async function getDependency( if (res.versions?.[version].deprecated) { release.isDeprecated = true; } - const source = getPackageSource(res.versions?.[version].repository); + const source = PackageSource.parse(res.versions?.[version].repository); if (source.sourceUrl && source.sourceUrl !== dep.sourceUrl) { release.sourceUrl = source.sourceUrl; } From 15a4c9947db62d3a9f417a4c1cdf5e1533f09aa7 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 19 May 2023 16:28:05 +0300 Subject: [PATCH 2/5] Fix for test --- lib/modules/datasource/npm/get.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 2a7ab6df9c2f88..44ddd035586e59 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -45,8 +45,8 @@ const PackageSource = z }), z .object({ - url: z.string().nonempty().optional(), - directory: z.string().nonempty().optional(), + url: z.string().nonempty().nullish(), + directory: z.string().nonempty().nullish(), }) .transform(({ url, directory }) => { const res: PackageSource = {}; From 92e1221a2a0cadd719d75d9cf8eff14d792e5e9b Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 19 May 2023 16:42:09 +0300 Subject: [PATCH 3/5] Make it nullish --- lib/modules/datasource/npm/get.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 44ddd035586e59..94109fcb1d2cd6 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -25,8 +25,8 @@ const platformMapping: Record = { }; interface PackageSource { - sourceUrl?: string; - sourceDirectory?: string; + sourceUrl: string | null; + sourceDirectory: string | null; } const PackageSource = z @@ -35,13 +35,16 @@ const PackageSource = z .string() .nonempty() .transform((repository): PackageSource => { + let sourceUrl: string | null = null; + const sourceDirectory = null; const shortMatch = repository.match(SHORT_REPO_REGEX); if (shortMatch?.groups) { const { platform = 'github', shortRepo } = shortMatch.groups; - return { sourceUrl: platformMapping[platform] + shortRepo }; + sourceUrl = platformMapping[platform] + shortRepo; } else { - return { sourceUrl: repository }; + sourceUrl = repository; } + return { sourceUrl, sourceDirectory }; }), z .object({ @@ -49,17 +52,20 @@ const PackageSource = z directory: z.string().nonempty().nullish(), }) .transform(({ url, directory }) => { - const res: PackageSource = {}; + const res: PackageSource = { sourceUrl: null, sourceDirectory: null }; + if (url) { res.sourceUrl = url; } + if (directory) { res.sourceDirectory = directory; } + return res; }), ]) - .catch({}); + .catch({ sourceUrl: null, sourceDirectory: null }); export async function getDependency( http: Http, @@ -141,13 +147,19 @@ export async function getDependency( // Simplify response before caching and returning const dep: ReleaseResult = { homepage: res.homepage, - sourceUrl, - sourceDirectory, releases: [], tags: res['dist-tags'], registryUrl, }; + if (sourceUrl) { + dep.sourceUrl = sourceUrl; + } + + if (sourceDirectory) { + dep.sourceDirectory = sourceDirectory; + } + if (latestVersion?.deprecated) { dep.deprecationMessage = `On registry \`${registryUrl}\`, the "latest" version of dependency \`${packageName}\` has the following deprecation notice:\n\n\`${latestVersion.deprecated}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`; } From e01b59c2651361050a3f59f967c4bfe8789ebc73 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 19 May 2023 17:57:35 +0300 Subject: [PATCH 4/5] refactor(npm): Use schema for datasource --- lib/modules/datasource/npm/get.ts | 204 +++++++++++++++++++++--------- 1 file changed, 146 insertions(+), 58 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 94109fcb1d2cd6..ed739f9d4a5f06 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -10,9 +10,10 @@ import * as packageCache from '../../../util/cache/package'; import type { Http } from '../../../util/http'; import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; +import { LooseRecord } from '../../../util/schema-utils'; import { joinUrlParts } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; -import type { CachedReleaseResult, NpmResponse } from './types'; +import type { CachedReleaseResult } from './types'; const SHORT_REPO_REGEX = regEx( /^((?bitbucket|github|gitlab):)?(?[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)$/ @@ -67,6 +68,140 @@ const PackageSource = z ]) .catch({ sourceUrl: null, sourceDirectory: null }); +const DepResponse = z + .object({ + versions: LooseRecord( + z.object({ + repository: PackageSource, + homepage: z.string().nullish().catch(null), + deprecated: z.union([z.boolean(), z.string()]).nullish().catch(null), + gitHead: z.string().nullish().catch(null), + dependencies: LooseRecord(z.string()).nullish().catch(null), + devDependencies: LooseRecord(z.string()).nullish().catch(null), + }) + ).catch({}), + repository: PackageSource, + homepage: z.string().nullish().catch(null), + time: LooseRecord(z.string()).catch({}), + 'dist-tags': LooseRecord(z.string()).nullish().catch(null), + }) + .transform((body) => { + const { time, versions, 'dist-tags': tags, repository } = body; + + const latestTag = tags?.latest; + const latestVersion = latestTag ? versions[latestTag] : undefined; + + const homepage = body.homepage ?? latestVersion?.homepage; + + const latestVersionRepository = latestVersion?.repository ?? repository; + const sourceUrl = repository.sourceUrl ?? latestVersionRepository.sourceUrl; + const sourceDirectory = + repository.sourceDirectory ?? latestVersionRepository.sourceDirectory; + + const deprecated = latestVersion?.deprecated; + let deprecationMessage: string | null = null; + if (is.boolean(deprecated)) { + deprecationMessage = 'deprecated by setting deprecated="true"'; + } else if (is.string(deprecated)) { + deprecationMessage = deprecated; + } + + return { + time, + versions, + tags, + latestTag, + latestVersion, + homepage, + sourceUrl, + sourceDirectory, + deprecationMessage, + }; + }) + .transform( + ({ + homepage, + sourceUrl, + sourceDirectory, + tags, + versions, + time, + deprecationMessage, + }): ReleaseResult | null => { + if (is.emptyObject(versions)) { + return null; + } + + const result: ReleaseResult = { releases: [] }; + + if (homepage) { + result.homepage = homepage; + } + + if (sourceUrl) { + result.sourceUrl = sourceUrl; + } + + if (sourceDirectory) { + result.sourceDirectory = sourceDirectory; + } + + if (tags) { + result.tags = tags; + } + + if (deprecationMessage) { + result.deprecationMessage = deprecationMessage; + } + + for (const [version, versionInfo] of Object.entries(versions)) { + const { + gitHead: gitRef, + dependencies, + devDependencies, + deprecated, + repository: src, + } = versionInfo; + + const release: Release = { version }; + + if (gitRef) { + release.gitRef = gitRef; + } + + if (dependencies) { + release.dependencies = dependencies; + } + + if (devDependencies) { + release.devDependencies = devDependencies; + } + + if (deprecated) { + release.isDeprecated = true; + } + + const releaseTimestamp = time[version]; + if (releaseTimestamp) { + release.releaseTimestamp = releaseTimestamp; + } + + if (src.sourceUrl && src.sourceUrl !== sourceUrl) { + release.sourceUrl = src.sourceUrl; + } + + if (src.sourceDirectory && src.sourceDirectory !== sourceDirectory) { + release.sourceDirectory = src.sourceDirectory; + } + + result.releases.push(release); + } + + return result; + } + ) + .catch(null); + export async function getDependency( http: Http, registryUrl: string, @@ -109,16 +244,14 @@ export async function getDependency( cacheHardTtlMinutes = cacheMinutes; } - const uri = url.parse(packageUrl); - try { const options: HttpOptions = {}; if (cachedResult?.cacheData?.etag) { logger.trace({ packageName }, 'Using cached etag'); options.headers = { 'If-None-Match': cachedResult.cacheData.etag }; } - const raw = await http.getJson(packageUrl, options); - if (cachedResult?.cacheData && raw.statusCode === 304) { + const res = await http.getJson(packageUrl, options, DepResponse); + if (cachedResult?.cacheData && res.statusCode === 304) { logger.trace(`Cached npm result for ${packageName} is revalidated`); cachedResult.cacheData.softExpireAt = softExpireAt; await packageCache.set( @@ -130,66 +263,20 @@ export async function getDependency( delete cachedResult.cacheData; return cachedResult; } - const etag = raw.headers.etag; - const res = raw.body; - if (!res.versions || !Object.keys(res.versions).length) { + const etag = res.headers.etag; + const dep = res.body; + if (!dep) { // Registry returned a 200 OK but with no versions logger.debug(`No versions returned for npm dependency ${packageName}`); return null; } - const latestVersion = res.versions[res['dist-tags']?.latest ?? '']; - res.repository ??= latestVersion?.repository; - res.homepage ??= latestVersion?.homepage; - - const { sourceUrl, sourceDirectory } = PackageSource.parse(res.repository); - - // Simplify response before caching and returning - const dep: ReleaseResult = { - homepage: res.homepage, - releases: [], - tags: res['dist-tags'], - registryUrl, - }; - - if (sourceUrl) { - dep.sourceUrl = sourceUrl; - } - - if (sourceDirectory) { - dep.sourceDirectory = sourceDirectory; + dep.registryUrl = registryUrl; + if (dep.deprecationMessage) { + dep.deprecationMessage = `On registry \`${registryUrl}\`, the "latest" version of dependency \`${packageName}\` has the following deprecation notice:\n\n\`${dep.deprecationMessage}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`; } - - if (latestVersion?.deprecated) { - dep.deprecationMessage = `On registry \`${registryUrl}\`, the "latest" version of dependency \`${packageName}\` has the following deprecation notice:\n\n\`${latestVersion.deprecated}\`\n\nMarking the latest version of an npm package as deprecated results in the entire package being considered deprecated, so contact the package author you think this is a mistake.`; - } - dep.releases = Object.keys(res.versions).map((version) => { - const release: Release = { - version, - gitRef: res.versions?.[version].gitHead, - dependencies: res.versions?.[version].dependencies, - devDependencies: res.versions?.[version].devDependencies, - }; - if (res.time?.[version]) { - release.releaseTimestamp = res.time[version]; - } - if (res.versions?.[version].deprecated) { - release.isDeprecated = true; - } - const source = PackageSource.parse(res.versions?.[version].repository); - if (source.sourceUrl && source.sourceUrl !== dep.sourceUrl) { - release.sourceUrl = source.sourceUrl; - } - if ( - source.sourceDirectory && - source.sourceDirectory !== dep.sourceDirectory - ) { - release.sourceDirectory = source.sourceDirectory; - } - return release; - }); logger.trace({ dep }, 'dep'); - const cacheControl = raw.headers?.['cache-control']; + const cacheControl = res.headers?.['cache-control']; if ( is.nonEmptyString(cacheControl) && regEx(/(^|,)\s*public\s*(,|$)/).test(cacheControl) @@ -216,6 +303,7 @@ export async function getDependency( ) { return null; } + const uri = url.parse(packageUrl); if (uri.host === 'registry.npmjs.org') { if (cachedResult) { logger.warn( From c48b080fb6ae9b7706a1d4de7afe28d30c25c896 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 19 May 2023 20:53:23 +0300 Subject: [PATCH 5/5] Fix deprecated field --- lib/modules/datasource/npm/get.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index ed739f9d4a5f06..44fe2529756279 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -74,7 +74,7 @@ const DepResponse = z z.object({ repository: PackageSource, homepage: z.string().nullish().catch(null), - deprecated: z.union([z.boolean(), z.string()]).nullish().catch(null), + deprecated: z.string().nullish().catch('Unknown deprecation reason'), gitHead: z.string().nullish().catch(null), dependencies: LooseRecord(z.string()).nullish().catch(null), devDependencies: LooseRecord(z.string()).nullish().catch(null), @@ -98,13 +98,7 @@ const DepResponse = z const sourceDirectory = repository.sourceDirectory ?? latestVersionRepository.sourceDirectory; - const deprecated = latestVersion?.deprecated; - let deprecationMessage: string | null = null; - if (is.boolean(deprecated)) { - deprecationMessage = 'deprecated by setting deprecated="true"'; - } else if (is.string(deprecated)) { - deprecationMessage = deprecated; - } + const deprecationMessage = latestVersion?.deprecated; return { time,