feat(datasource/crate): fetch crate metadata from crates.io (#15214)

* feat(datasource/crate): fetch crate metadata from crates.io

* refactor(datasource/crate): simplify `mockCratesApiCallFor()` function

Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/datasource/crate/__fixtures__/amethyst.json

@@ -0,0 +1,33 @@
+{
+  "categories": null,
+  "crate": {
+    "badges": null,
+    "categories": null,
+    "created_at": "2016-01-04T03:42:04.120616+00:00",
+    "description": "Data-oriented game engine written in Rust",
+    "documentation": "https://docs.amethyst.rs/stable/amethyst",
+    "downloads": 121814,
+    "exact_match": false,
+    "homepage": "https://amethyst.rs/",
+    "id": "amethyst",
+    "keywords": null,
+    "links": {
+      "owner_team": "/api/v1/crates/amethyst/owner_team",
+      "owner_user": "/api/v1/crates/amethyst/owner_user",
+      "owners": "/api/v1/crates/amethyst/owners",
+      "reverse_dependencies": "/api/v1/crates/amethyst/reverse_dependencies",
+      "version_downloads": "/api/v1/crates/amethyst/downloads",
+      "versions": "/api/v1/crates/amethyst/versions"
+    },
+    "max_stable_version": null,
+    "max_version": "0.0.0",
+    "name": "amethyst",
+    "newest_version": "0.0.0",
+    "recent_downloads": null,
+    "repository": "https://github.com/amethyst/amethyst",
+    "updated_at": "2020-08-23T02:18:12.559064+00:00",
+    "versions": null
+  },
+  "keywords": null,
+  "versions": null
+}

lib/modules/datasource/crate/__fixtures__/libc.json

@@ -0,0 +1,33 @@
+{
+  "categories": null,
+  "crate": {
+    "badges": null,
+    "categories": null,
+    "created_at": "2015-01-15T20:22:13.100871+00:00",
+    "description": "Raw FFI bindings to platform libraries like libc.\n",
+    "documentation": "https://docs.rs/libc/",
+    "downloads": 106751998,
+    "exact_match": false,
+    "homepage": "https://github.com/rust-lang/libc",
+    "id": "libc",
+    "keywords": null,
+    "links": {
+      "owner_team": "/api/v1/crates/libc/owner_team",
+      "owner_user": "/api/v1/crates/libc/owner_user",
+      "owners": "/api/v1/crates/libc/owners",
+      "reverse_dependencies": "/api/v1/crates/libc/reverse_dependencies",
+      "version_downloads": "/api/v1/crates/libc/downloads",
+      "versions": "/api/v1/crates/libc/versions"
+    },
+    "max_stable_version": null,
+    "max_version": "0.0.0",
+    "name": "libc",
+    "newest_version": "0.0.0",
+    "recent_downloads": null,
+    "repository": "https://github.com/rust-lang/libc",
+    "updated_at": "2022-04-18T23:40:23.878438+00:00",
+    "versions": null
+  },
+  "keywords": null,
+  "versions": null
+}

lib/modules/datasource/crate/__snapshots__/index.spec.ts.snap

@@ -33,6 +33,7 @@ Object {
 exports[`modules/datasource/crate/index getReleases processes real data: amethyst 1`] = `
 Object {
   "dependencyUrl": "https://crates.io/crates/amethyst",
+  "homepage": "https://amethyst.rs/",
   "registryUrl": "https://crates.io",
   "releases": Array [
     Object {
@@ -94,6 +95,7 @@ Object {
       "version": "0.10.1",
     },
   ],
+  "sourceUrl": "https://github.com/amethyst/amethyst",
 }
 `;
 
@@ -300,6 +302,7 @@ Object {
       "version": "0.2.51",
     },
   ],
+  "sourceUrl": "https://github.com/rust-lang/libc",
 }
 `;
 

lib/modules/datasource/crate/index.spec.ts

@@ -15,6 +15,8 @@ import { CrateDatasource } from '.';
 jest.mock('simple-git');
 const simpleGit: jest.Mock<Partial<SimpleGit>> = _simpleGit as never;
 
+const API_BASE_URL = CrateDatasource.CRATES_IO_API_BASE_URL;
+
 const baseUrl =
   'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
 
@@ -58,6 +60,13 @@ function setupErrorGitMock(): { mockClone: jest.Mock<any, any> } {
   return { mockClone };
 }
 
+function mockCratesApiCallFor(crateName: string, response?: httpMock.Body) {
+  httpMock
+    .scope(API_BASE_URL)
+    .get(`/crates/${crateName}?include=`)
+    .reply(response ? 200 : 404, response);
+}
+
 describe('modules/datasource/crate/index', () => {
   describe('getIndexSuffix', () => {
     it('returns correct suffixes', () => {
@@ -133,6 +142,7 @@ describe('modules/datasource/crate/index', () => {
     });
 
     it('returns null for empty result', async () => {
+      mockCratesApiCallFor('non_existent_crate');
       httpMock.scope(baseUrl).get('/no/n_/non_existent_crate').reply(200, {});
       expect(
         await getPkgReleases({
@@ -144,6 +154,7 @@ describe('modules/datasource/crate/index', () => {
     });
 
     it('returns null for missing fields', async () => {
+      mockCratesApiCallFor('non_existent_crate');
       httpMock
         .scope(baseUrl)
         .get('/no/n_/non_existent_crate')
@@ -158,6 +169,7 @@ describe('modules/datasource/crate/index', () => {
     });
 
     it('returns null for empty list', async () => {
+      mockCratesApiCallFor('non_existent_crate');
       httpMock.scope(baseUrl).get('/no/n_/non_existent_crate').reply(200, '\n');
       expect(
         await getPkgReleases({
@@ -207,6 +219,8 @@ describe('modules/datasource/crate/index', () => {
     });
 
     it('processes real data: libc', async () => {
+      mockCratesApiCallFor('libc', Fixtures.get('libc.json'));
+
       httpMock
         .scope(baseUrl)
         .get('/li/bc/libc')
@@ -222,6 +236,8 @@ describe('modules/datasource/crate/index', () => {
     });
 
     it('processes real data: amethyst', async () => {
+      mockCratesApiCallFor('amethyst', Fixtures.get('amethyst.json'));
+
       httpMock
         .scope(baseUrl)
         .get('/am/et/amethyst')

lib/modules/datasource/crate/index.ts

@@ -12,7 +12,12 @@ import { parseUrl } from '../../../util/url';
 import * as cargoVersioning from '../../versioning/cargo';
 import { Datasource } from '../datasource';
 import type { GetReleasesConfig, Release, ReleaseResult } from '../types';
-import { CrateRecord, RegistryFlavor, RegistryInfo } from './types';
+import {
+  CrateMetadata,
+  CrateRecord,
+  RegistryFlavor,
+  RegistryInfo,
+} from './types';
 
 export class CrateDatasource extends Datasource {
   static readonly id = 'crate';
@@ -28,6 +33,8 @@ export class CrateDatasource extends Datasource {
   static readonly CRATES_IO_BASE_URL =
     'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
 
+  static readonly CRATES_IO_API_BASE_URL = 'https://crates.io/api/v1/';
+
   @cache({
     namespace: `datasource-${CrateDatasource.id}`,
     key: ({ registryUrl, packageName }: GetReleasesConfig) =>
@@ -70,10 +77,22 @@ export class CrateDatasource extends Datasource {
       .map((line) => line.trim()) // remove whitespace
       .filter((line) => line.length !== 0) // remove empty lines
       .map((line) => JSON.parse(line) as CrateRecord); // parse
+
+    const metadata = await this.getCrateMetadata(registryInfo, packageName);
+
     const result: ReleaseResult = {
       dependencyUrl,
       releases: [],
     };
+
+    if (metadata?.homepage) {
+      result.homepage = metadata.homepage;
+    }
+
+    if (metadata?.repository) {
+      result.sourceUrl = metadata.repository;
+    }
+
     result.releases = lines
       .map((version) => {
         const release: Release = {
@@ -92,6 +111,46 @@ export class CrateDatasource extends Datasource {
     return result;
   }
 
+  @cache({
+    namespace: `datasource-${CrateDatasource.id}-metadata`,
+    key: (info: RegistryInfo, packageName: string) =>
+      `${info.rawUrl}/${packageName}`,
+    cacheable: (info: RegistryInfo) =>
+      CrateDatasource.areReleasesCacheable(info.rawUrl),
+    ttlMinutes: 24 * 60, // 24 hours
+  })
+  public async getCrateMetadata(
+    info: RegistryInfo,
+    packageName: string
+  ): Promise<CrateMetadata | null> {
+    if (info.flavor !== RegistryFlavor.CratesIo) {
+      return null;
+    }
+
+    // The `?include=` suffix is required to avoid unnecessary database queries
+    // on the crates.io server. This lets us work around the regular request
+    // throttling of one request per second.
+    const crateUrl = `${CrateDatasource.CRATES_IO_API_BASE_URL}crates/${packageName}?include=`;
+
+    logger.debug(
+      { crateUrl, packageName, registryUrl: info.rawUrl },
+      'downloading crate metadata'
+    );
+
+    try {
+      type Response = { crate: CrateMetadata };
+      const response = await this.http.getJson<Response>(crateUrl);
+      return response.body.crate;
+    } catch (err) {
+      logger.warn(
+        { err, packageName, registryUrl: info.rawUrl },
+        'failed to download crate metadata'
+      );
+    }
+
+    return null;
+  }
+
   public async fetchCrateRecordsPayload(
     info: RegistryInfo,
     packageName: string

lib/modules/datasource/crate/types.ts

@@ -27,3 +27,10 @@ export interface CrateRecord {
   vers: string;
   yanked: boolean;
 }
+
+export interface CrateMetadata {
+  description: string | null;
+  documentation: string | null;
+  homepage: string | null;
+  repository: string | null;
+}

test/http-mock.ts

@@ -5,7 +5,7 @@ import nock from 'nock';
 import { makeGraphqlSnapshot } from './graphql-snapshot';
 
 // eslint-disable-next-line no-restricted-imports
-export type { Scope, ReplyHeaders } from 'nock';
+export type { Scope, ReplyHeaders, Body } from 'nock';
 
 interface RequestLogItem {
   headers: Record<string, string>;