feat(datasource): implement custom datasource (#23147)

docs/usage/configuration-options.md

@@ -633,6 +633,38 @@ When using with `npm`, we recommend you:
 - Use `constraintsFiltering` on `dependencies`, not `devDependencies` (usually you do not need to be strict about development dependencies)
 - Do _not_ enable `rollbackPrs` at the same time (otherwise your _current_ version may be rolled back if it's incompatible)
 
+## customDatasources
+
+Use `customDatasources` to fetch releases from APIs or statically hosted sites and Renovate has no own datasource.
+These datasources can be referred by RegexManagers or can be used to overwrite default datasources.
+
+For more details see the [`custom` datasource documentation](/modules/datasource/custom/).
+
+### defaultRegistryUrlTemplate
+
+`registryUrl` which is used, if none is return by extraction.
+As this is a template it can be dynamically set. E.g. add the `packageName` as part of the URL:
+
+```json5
+{
+  customDatasources: {
+    foo: {
+      defaultRegistryUrlTemplate: 'https://exmaple.foo.bar/v1/{{ packageName }}',
+    },
+  },
+}
+```
+
+### format
+
+Defines which format the API is returning.
+Only `json` is supported, but more are planned for future.
+
+### transformTemplates
+
+`transformTemplates` is a list of [jsonata rules](https://docs.jsonata.org/simple) which get applied serially.
+Use this if the API does not return a Renovate compatible schema.
+
 ## defaultRegistryUrls
 
 Override a datasource's default registries with this config option.

lib/config/options/index.ts

@@ -68,6 +68,16 @@ const options: RenovateOptions[] = [
     default: ['**/*'],
     cli: false,
   },
+  {
+    name: 'format',
+    description: 'Format of the custom datasource',
+    type: 'string',
+    parent: 'customDatasources',
+    default: 'json',
+    allowedValues: ['json'],
+    cli: false,
+    env: false,
+  },
   {
     name: 'executionMode',
     description:
@@ -334,6 +344,13 @@ const options: RenovateOptions[] = [
     type: 'object',
     default: {},
   },
+  {
+    name: 'customDatasources',
+    description: 'Defines custom datasources for usage by managers',
+    type: 'object',
+    experimental: true,
+    default: {},
+  },
   {
     name: 'dockerChildPrefix',
     description:
@@ -932,6 +949,16 @@ const options: RenovateOptions[] = [
     cli: false,
     env: false,
   },
+  {
+    name: 'defaultRegistryUrlTemplate',
+    description:
+      'Template for generating a defaultRegistryUrl for custom datasource',
+    type: 'string',
+    default: '',
+    parent: 'customDatasources',
+    cli: false,
+    env: false,
+  },
   {
     name: 'registryUrls',
     description:
@@ -1750,6 +1777,14 @@ const options: RenovateOptions[] = [
     type: 'boolean',
     default: false,
   },
+  {
+    name: 'transformTemplates',
+    description: 'List of jsonata transformation rules',
+    type: 'array',
+    subType: 'string',
+    parent: 'customDatasources',
+    default: [],
+  },
   {
     name: 'transitiveRemediation',
     description: 'Enable remediation of transitive dependencies.',

lib/config/types.ts

@@ -261,6 +261,7 @@ export interface RenovateConfig
   osvVulnerabilityAlerts?: boolean;
   vulnerabilitySeverity?: string;
   regexManagers?: RegExManager[];
+  customDatasources?: Record<string, CustomDatasourceConfig>;
 
   fetchReleaseNotes?: FetchReleaseNotesOptions;
   secrets?: Record<string, string>;
@@ -273,6 +274,12 @@ export interface RenovateConfig
   checkedBranches?: string[];
 }
 
+export interface CustomDatasourceConfig {
+  defaultRegistryUrlTemplate?: string;
+  format?: 'json';
+  transformTemplates?: string[];
+}
+
 export interface AllConfig
   extends RenovateConfig,
     GlobalOnlyConfig,
@@ -380,7 +387,12 @@ export interface RenovateOptionBase {
 
   name: string;
 
-  parent?: 'hostRules' | 'packageRules' | 'postUpgradeTasks' | 'regexManagers';
+  parent?:
+    | 'customDatasources'
+    | 'hostRules'
+    | 'packageRules'
+    | 'postUpgradeTasks'
+    | 'regexManagers';
 
   // used by tests
   relatedOptions?: string[];

lib/config/validation.spec.ts

@@ -117,6 +117,48 @@ describe('config/validation', () => {
       expect(errors).toMatchSnapshot();
     });
 
+    it('catches invalid customDatasources content', async () => {
+      const config = {
+        customDatasources: {
+          foo: {
+            randomKey: '',
+            defaultRegistryUrlTemplate: [],
+            transformTemplates: [{}],
+          },
+        },
+      } as any;
+      const { errors } = await configValidation.validateConfig(config);
+      expect(errors).toMatchObject([
+        {
+          message:
+            'Invalid `customDatasources.customDatasources.defaultRegistryUrlTemplate` configuration: is a string',
+        },
+        {
+          message:
+            'Invalid `customDatasources.customDatasources.randomKey` configuration: key is not allowed',
+        },
+        {
+          message:
+            'Invalid `customDatasources.customDatasources.transformTemplates` configuration: is not an array of string',
+        },
+      ]);
+    });
+
+    it('catches invalid customDatasources record type', async () => {
+      const config = {
+        customDatasources: {
+          randomKey: '',
+        },
+      } as any;
+      const { errors } = await configValidation.validateConfig(config);
+      expect(errors).toMatchObject([
+        {
+          message:
+            'Invalid `customDatasources.randomKey` configuration: customDatasource is not an object',
+        },
+      ]);
+    });
+
     it('catches invalid baseBranches regex', async () => {
       const config = {
         baseBranches: ['/***$}{]][/'],

lib/config/validation.ts

@@ -568,6 +568,47 @@ export async function validateConfig(
                   message: `Invalid \`${currentPath}.${key}.${res}\` configuration: value is not a string`,
                 });
               }
+            } else if (key === 'customDatasources') {
+              const allowedKeys = [
+                'description',
+                'defaultRegistryUrlTemplate',
+                'format',
+                'transformTemplates',
+              ];
+              for (const [
+                customDatasourceName,
+                customDatasourceValue,
+              ] of Object.entries(val)) {
+                if (!is.plainObject(customDatasourceValue)) {
+                  errors.push({
+                    topic: 'Configuration Error',
+                    message: `Invalid \`${currentPath}.${customDatasourceName}\` configuration: customDatasource is not an object`,
+                  });
+                  continue;
+                }
+                for (const [subKey, subValue] of Object.entries(
+                  customDatasourceValue
+                )) {
+                  if (!allowedKeys.includes(subKey)) {
+                    errors.push({
+                      topic: 'Configuration Error',
+                      message: `Invalid \`${currentPath}.${key}.${subKey}\` configuration: key is not allowed`,
+                    });
+                  } else if (subKey === 'transformTemplates') {
+                    if (!is.array(subValue, is.string)) {
+                      errors.push({
+                        topic: 'Configuration Error',
+                        message: `Invalid \`${currentPath}.${key}.${subKey}\` configuration: is not an array of string`,
+                      });
+                    }
+                  } else if (!is.string(subValue)) {
+                    errors.push({
+                      topic: 'Configuration Error',
+                      message: `Invalid \`${currentPath}.${key}.${subKey}\` configuration: is a string`,
+                    });
+                  }
+                }
+              }
             } else if (
               ['customEnvVariables', 'migratePresets', 'secrets'].includes(key)
             ) {

lib/modules/datasource/api.ts

@@ -11,6 +11,7 @@ import { ConanDatasource } from './conan';
 import { CondaDatasource } from './conda';
 import { CpanDatasource } from './cpan';
 import { CrateDatasource } from './crate';
+import { CustomDatasource } from './custom';
 import { DartDatasource } from './dart';
 import { DartVersionDatasource } from './dart-version';
 import { DenoDatasource } from './deno';
@@ -72,6 +73,7 @@ api.set(ConanDatasource.id, new ConanDatasource());
 api.set(CondaDatasource.id, new CondaDatasource());
 api.set(CpanDatasource.id, new CpanDatasource());
 api.set(CrateDatasource.id, new CrateDatasource());
+api.set(CustomDatasource.id, new CustomDatasource());
 api.set(DartDatasource.id, new DartDatasource());
 api.set(DartVersionDatasource.id, new DartVersionDatasource());
 api.set(DenoDatasource.id, new DenoDatasource());