feat(datasource/azure-pipelines-tasks): add azure-pipelines-tasks datasource

lib/modules/datasource/api.ts

@@ -1,6 +1,7 @@
 import { AdoptiumJavaDatasource } from './adoptium-java';
 import { ArtifactoryDatasource } from './artifactory';
 import { AwsMachineImageDataSource } from './aws-machine-image';
+import { AzurePipelinesTasksDatasource } from './azure-pipelines-tasks';
 import { BitBucketTagsDatasource } from './bitbucket-tags';
 import { CdnJsDatasource } from './cdnjs';
 import { ClojureDatasource } from './clojure';
@@ -51,6 +52,7 @@ export default api;
 api.set(AdoptiumJavaDatasource.id, new AdoptiumJavaDatasource());
 api.set(ArtifactoryDatasource.id, new ArtifactoryDatasource());
 api.set(AwsMachineImageDataSource.id, new AwsMachineImageDataSource());
+api.set(AzurePipelinesTasksDatasource.id, new AzurePipelinesTasksDatasource());
 api.set(BitBucketTagsDatasource.id, new BitBucketTagsDatasource());
 api.set(CdnJsDatasource.id, new CdnJsDatasource());
 api.set(ClojureDatasource.id, new ClojureDatasource());

lib/modules/datasource/azure-pipelines-tasks/index.spec.ts

@@ -0,0 +1,31 @@
+import { getPkgReleases } from '../index';
+import { AzurePipelinesTasksDatasource } from '.';
+
+describe('modules/datasource/azure-pipelines-tasks/index', () => {
+  it('returns null for unknown task', async () => {
+    expect(
+      await getPkgReleases({
+        datasource: AzurePipelinesTasksDatasource.id,
+        depName: 'unknown',
+      })
+    ).toBeNull();
+  });
+
+  it('supports built-in tasks', async () => {
+    expect(
+      await getPkgReleases({
+        datasource: AzurePipelinesTasksDatasource.id,
+        depName: 'AutomatedAnalysis',
+      })
+    ).toEqual({ releases: [{ version: '0.171.0' }, { version: '0.198.0' }] });
+  });
+
+  it('is case insensitive', async () => {
+    expect(
+      await getPkgReleases({
+        datasource: AzurePipelinesTasksDatasource.id,
+        depName: 'automatedanalysis',
+      })
+    ).toEqual({ releases: [{ version: '0.171.0' }, { version: '0.198.0' }] });
+  });
+});

lib/modules/datasource/azure-pipelines-tasks/index.ts

@@ -0,0 +1,35 @@
+import dataFiles from '../../../data-files.generated';
+import { id as versioning } from '../../versioning/loose';
+import { Datasource } from '../datasource';
+import type { GetReleasesConfig, ReleaseResult } from '../types';
+
+export class AzurePipelinesTasksDatasource extends Datasource {
+  static readonly id = 'azure-pipelines-tasks';
+
+  private readonly builtInTasks: Record<string, string[]>;
+
+  constructor() {
+    super(AzurePipelinesTasksDatasource.id);
+    this.builtInTasks = JSON.parse(
+      dataFiles.get('data/azure-pipelines-tasks.json')!
+    );
+  }
+
+  override readonly caching = true;
+
+  override readonly customRegistrySupport = false;
+
+  override readonly defaultVersioning = versioning;
+
+  getReleases({
+    packageName,
+  }: GetReleasesConfig): Promise<ReleaseResult | null> {
+    const versions = this.builtInTasks[packageName.toLowerCase()];
+    if (versions) {
+      const releases = versions.map((version) => ({ version }));
+      return Promise.resolve({ releases });
+    }
+
+    return Promise.resolve(null);
+  }
+}

lib/modules/datasource/azure-pipelines-tasks/readme.md

@@ -0,0 +1,2 @@
+This datasource returns versions of the [built-in Azure Pipelines tasks](https://docs.microsoft.com/en-us/azure/devops/pipelines/tasks/?view=azure-devops).
+It does not yet support Azure Pipelines tasks from the [Visual Studio Marketplace](https://marketplace.visualstudio.com/search?target=AzureDevOps&category=Azure%20Pipelines)

lib/modules/manager/azure-pipelines/extract.spec.ts

@@ -1,4 +1,5 @@
 import { Fixtures } from '../../../../test/fixtures';
+import { AzurePipelinesTasksDatasource } from '../../datasource/azure-pipelines-tasks';
 import {
   extractAzurePipelinesTasks,
   extractContainer,
@@ -96,6 +97,7 @@ describe('modules/manager/azure-pipelines/extract', () => {
       expect(extractAzurePipelinesTasks('Bash@3')).toEqual({
         depName: 'Bash',
         currentValue: '3',
+        datasource: AzurePipelinesTasksDatasource.id,
       });
     });
 
@@ -142,23 +144,41 @@ describe('modules/manager/azure-pipelines/extract', () => {
         azurePipelinesStages,
         azurePipelinesFilename
       );
-      expect(res?.deps).toEqual([{ depName: 'Bash', currentValue: '3' }]);
+      expect(res?.deps).toEqual([
+        {
+          depName: 'Bash',
+          currentValue: '3',
+          datasource: AzurePipelinesTasksDatasource.id,
+        },
+      ]);
     });
 
     it('should extract jobs', () => {
       const res = extractPackageFile(
         azurePipelinesJobs,
         azurePipelinesFilename
       );
-      expect(res?.deps).toEqual([{ depName: 'Bash', currentValue: '3' }]);
+      expect(res?.deps).toEqual([
+        {
+          depName: 'Bash',
+          currentValue: '3',
+          datasource: AzurePipelinesTasksDatasource.id,
+        },
+      ]);
     });
 
     it('should extract steps', () => {
       const res = extractPackageFile(
         azurePipelinesSteps,
         azurePipelinesFilename
       );
-      expect(res?.deps).toEqual([{ depName: 'Bash', currentValue: '3' }]);
+      expect(res?.deps).toEqual([
+        {
+          depName: 'Bash',
+          currentValue: '3',
+          datasource: AzurePipelinesTasksDatasource.id,
+        },
+      ]);
     });
 
     it('should return null when task alias used', () => {

lib/modules/manager/azure-pipelines/extract.ts

@@ -1,6 +1,7 @@
 import { load } from 'js-yaml';
 import { logger } from '../../../logger';
 import { regEx } from '../../../util/regex';
+import { AzurePipelinesTasksDatasource } from '../../datasource/azure-pipelines-tasks';
 import { GitTagsDatasource } from '../../datasource/git-tags';
 import { getDep } from '../dockerfile/extract';
 import type { PackageDependency, PackageFile } from '../types';
@@ -59,6 +60,7 @@ export function extractAzurePipelinesTasks(
     return {
       depName: match.groups.name,
       currentValue: match.groups.version,
+      datasource: AzurePipelinesTasksDatasource.id,
     };
   }
   return null;

lib/modules/manager/azure-pipelines/index.ts

@@ -1,8 +1,12 @@
+import { AzurePipelinesTasksDatasource } from '../../datasource/azure-pipelines-tasks';
 import { GitTagsDatasource } from '../../datasource/git-tags';
 export { extractPackageFile } from './extract';
 
 export const defaultConfig = {
   fileMatch: ['azure.*pipelines?.*\\.ya?ml$'],
 };
 
-export const supportedDatasources = [GitTagsDatasource.id];
+export const supportedDatasources = [
+  AzurePipelinesTasksDatasource.id,
+  GitTagsDatasource.id,
+];

package.json

@@ -52,6 +52,7 @@
     "tsc": "tsc",
     "type-check": "run-s generate:* \"tsc --noEmit {@}\" --",
     "update:distro-info": "node tools/distro-json-generate.mjs",
+    "update:azure-pipelines-tasks": "node tools/azure-pipelines-tasks-json-generate.mjs",
     "verify": "node tools/verify.mjs"
   },
   "repository": {

tools/azure-pipelines-tasks-json-generate.mjs

@@ -0,0 +1,70 @@
+import os from 'os';
+import { promisify } from 'util';
+import fs from 'fs-extra';
+import g from 'glob';
+import JSON5 from 'json5';
+import shell from 'shelljs';
+import Git from 'simple-git';
+import path from 'upath';
+
+const glob = promisify(g);
+const localPath = path.join(os.tmpdir(), 'azure-pipelines-tasks');
+
+/**
+ * This script:
+ *  1. Clones the Azure Pipelines Tasks repo
+ *  2. Finds all `task.json` files
+ *  3. For each `task.json` it finds each commit that has that file
+ *  4. For each commit it gets the `task.json` content and extracts the task name and version
+ *  5. After all the `task.json` files have been processed it writes the results to `./data/azure-pipelines-tasks.json`
+ */
+await (async () => {
+  await fs.ensureDir(localPath);
+  const git = Git(localPath);
+
+  if (await git.checkIsRepo()) {
+    await git.pull();
+  } else {
+    await git.clone(
+      'https://github.com/microsoft/azure-pipelines-tasks.git',
+      '.'
+    );
+  }
+
+  // Find all `task.json` files
+  const files = (await glob(path.join(localPath, '**/task.json'))).map((file) =>
+    file.replace(`${localPath}/`, '')
+  );
+
+  /** @type {Record<string, Set<string>>} */
+  const tasks = {};
+
+  for (const file of files) {
+    // Find all commits that have the file
+    const revs = (await git.raw(['rev-list', 'HEAD', '--', file])).split('\n');
+    shell.echo(`Parsing ${file}`);
+    for (const rev of revs) {
+      try {
+        // Get the content of the file at the commit
+        const content = await git.show([`${rev}:${file}`]);
+        /** @type {{name: string, version: {Major: number, Minor: number, Patch: number}}} */
+        const parsedContent = JSON5.parse(content);
+        const version = `${parsedContent.version.Major}.${parsedContent.version.Minor}.${parsedContent.version.Patch}`;
+        tasks[parsedContent.name.toLowerCase()] =
+          tasks[parsedContent.name.toLowerCase()]?.add(version) ??
+          new Set([version]);
+      } catch (e) {
+        shell.echo(`Failed to parse ${file} at ${rev}`);
+        shell.echo(e.toString());
+      }
+    }
+  }
+
+  const data = JSON.stringify(
+    tasks,
+    (_, value) => (value instanceof Set ? [...value] : value),
+    2
+  );
+
+  await fs.writeFile(`./data/azure-pipelines-tasks.json`, data);
+})();