fix(core/http): fallback to github /gitlab hosttype

lib/types/github.spec.ts → lib/constants/platform.spec.ts

@@ -1,13 +1,32 @@
-import {
-  PLATFORM_TYPE_GITHUB,
-  PLATFORM_TYPE_GITLAB,
-} from '../constants/platforms';
 import { id as GH_RELEASES_DS } from '../datasource/github-releases';
 import { id as GH_TAGS_DS } from '../datasource/github-tags';
+import { GitlabReleasesDatasource } from '../datasource/gitlab-releases';
+import { id as GL_TAGS_DS } from '../datasource/gitlab-tags';
 import { id as POD_DS } from '../datasource/pod';
-import { GITHUB_API_USING_HOST_TYPES } from './github';
+import {
+  GITHUB_API_USING_HOST_TYPES,
+  GITLAB_API_USING_HOST_TYPES,
+  PLATFORM_TYPE_GITHUB,
+  PLATFORM_TYPE_GITLAB,
+} from './platforms';
+
+describe('constants/platform', () => {
+  it('should be part of the GITLAB_API_USING_HOST_TYPES', () => {
+    expect(GITLAB_API_USING_HOST_TYPES.includes(GL_TAGS_DS)).toBeTrue();
+    expect(
+      GITLAB_API_USING_HOST_TYPES.includes(GitlabReleasesDatasource.id)
+    ).toBeTrue();
+    expect(
+      GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITLAB)
+    ).toBeTrue();
+  });
+
+  it('should be not part of the GITLAB_API_USING_HOST_TYPES ', () => {
+    expect(
+      GITLAB_API_USING_HOST_TYPES.includes(PLATFORM_TYPE_GITHUB)
+    ).toBeFalse();
+  });
 
-describe('types/github', () => {
   it('should be part of the GITHUB_API_USING_HOST_TYPES ', () => {
     expect(GITHUB_API_USING_HOST_TYPES.includes(GH_TAGS_DS)).toBeTrue();
     expect(GITHUB_API_USING_HOST_TYPES.includes(GH_RELEASES_DS)).toBeTrue();

lib/constants/platforms.ts

@@ -4,3 +4,16 @@ export const PLATFORM_TYPE_BITBUCKET_SERVER = 'bitbucket-server';
 export const PLATFORM_TYPE_GITEA = 'gitea';
 export const PLATFORM_TYPE_GITHUB = 'github';
 export const PLATFORM_TYPE_GITLAB = 'gitlab';
+
+export const GITHUB_API_USING_HOST_TYPES = [
+  PLATFORM_TYPE_GITHUB,
+  'github-releases',
+  'github-tags',
+  'pod',
+];
+
+export const GITLAB_API_USING_HOST_TYPES = [
+  PLATFORM_TYPE_GITLAB,
+  'gitlab-releases',
+  'gitlab-tags',
+];

lib/platform/github/__snapshots__/index.spec.ts.snap

@@ -6875,33 +6875,6 @@ Array [
 ]
 `;
 
-exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 1`] = `
-Object {
-  "endpoint": "https://api.github.com/",
-  "gitAuthor": "renovate@whitesourcesoftware.com",
-  "renovateUsername": "renovate-bot",
-}
-`;
-
-exports[`platform/github/index initPlatform() should add initialized platform with predefined generic host rule for github api 2`] = `
-[MockFunction] {
-  "calls": Array [
-    Array [
-      Object {
-        "matchHost": "api.github.com",
-        "token": "abc123",
-      },
-    ],
-  ],
-  "results": Array [
-    Object {
-      "type": "return",
-      "value": undefined,
-    },
-  ],
-}
-`;
-
 exports[`platform/github/index initPlatform() should support custom endpoint 1`] = `
 Object {
   "endpoint": "https://ghe.renovatebot.com/",

lib/platform/github/index.spec.ts

@@ -129,17 +129,6 @@ describe('platform/github/index', () => {
       ).toMatchSnapshot();
       expect(httpMock.getTrace()).toMatchSnapshot();
     });
-
-    it('should add initialized platform with predefined generic host rule for github api', async () => {
-      expect(
-        await github.initPlatform({
-          token: 'abc123',
-          username: 'renovate-bot',
-          gitAuthor: 'renovate@whitesourcesoftware.com',
-        } as any)
-      ).toMatchSnapshot();
-      expect(hostRules.add).toMatchSnapshot();
-    });
   });
 
   describe('getRepos', () => {

lib/platform/github/index.ts

@@ -23,7 +23,7 @@ import * as git from '../../util/git';
 import * as hostRules from '../../util/host-rules';
 import * as githubHttp from '../../util/http/github';
 import { sanitize } from '../../util/sanitize';
-import { ensureTrailingSlash, parseUrl } from '../../util/url';
+import { ensureTrailingSlash } from '../../util/url';
 import type {
   AggregatedVulnerabilities,
   BranchStatusConfig,
@@ -115,12 +115,6 @@ export async function initPlatform({
     renovateUsername,
   };
 
-  // Generic github hostRule that per default all datasources using github api are enabled
-  const genericGithubHostRule = {
-    matchHost: parseUrl(defaults.endpoint).hostname,
-    token,
-  };
-  hostRules.add(genericGithubHostRule);
   return platformConfig;
 }
 

lib/types/index.ts

@@ -4,4 +4,3 @@ export * from './versioning';
 export * from './branch-status';
 export * from './vulnerability-alert';
 export * from './pr-state';
-export * from './github';

lib/types/platform/gitlab/index.ts

@@ -1,4 +1,3 @@
-import { PLATFORM_TYPE_GITLAB } from '../../../constants/platforms';
 import { GitTreeNode } from '../../git';
 
 export type GitLabBranch = {
@@ -13,9 +12,3 @@ export type GitlabTreeNode = {
   id: string;
   name: string;
 } & GitTreeNode;
-
-export const GITLAB_API_USING_HOST_TYPES = [
-  PLATFORM_TYPE_GITLAB,
-  'gitlab-releases',
-  'gitlab-tags',
-];

lib/util/http/auth.ts

@@ -1,9 +1,11 @@
 import is from '@sindresorhus/is';
-import { NormalizedOptions } from 'got';
-import { PLATFORM_TYPE_GITEA } from '../../constants/platforms';
-import { GITHUB_API_USING_HOST_TYPES } from '../../types';
-import { GITLAB_API_USING_HOST_TYPES } from '../../types/platform/gitlab';
-import { GotOptions } from './types';
+import type { NormalizedOptions } from 'got';
+import {
+  GITHUB_API_USING_HOST_TYPES,
+  GITLAB_API_USING_HOST_TYPES,
+  PLATFORM_TYPE_GITEA,
+} from '../../constants/platforms';
+import type { GotOptions } from './types';
 
 export function applyAuthorization(inOptions: GotOptions): GotOptions {
   const options: GotOptions = { ...inOptions };

lib/util/http/host-rules.spec.ts

@@ -36,6 +36,11 @@ describe('util/http/host-rules', () => {
       authType: 'Basic',
       token: 'XXX',
     });
+
+    hostRules.add({
+      hostType: 'gitlab-tags',
+      token: 'abc',
+    });
   });
 
   afterEach(() => {
@@ -111,4 +116,40 @@ describe('util/http/host-rules', () => {
       }
     `);
   });
+
+  it('noAuth', () => {
+    expect(applyHostRules(url, { ...options, noAuth: true }))
+      .toMatchInlineSnapshot(`
+      Object {
+        "hostType": "github",
+        "noAuth": true,
+      }
+    `);
+  });
+
+  it('fallback to github', () => {
+    expect(applyHostRules(url, { ...options, hostType: 'github-tags' }))
+      .toMatchInlineSnapshot(`
+      Object {
+        "context": Object {
+          "authType": undefined,
+        },
+        "hostType": "github-tags",
+        "token": "token",
+      }
+    `);
+  });
+
+  it('fallback to gitlab', () => {
+    expect(applyHostRules(url, { ...options, hostType: 'gitlab-tags' }))
+      .toMatchInlineSnapshot(`
+      Object {
+        "context": Object {
+          "authType": undefined,
+        },
+        "hostType": "gitlab-tags",
+        "token": "abc",
+      }
+    `);
+  });
 });

lib/util/http/host-rules.ts

@@ -1,17 +1,54 @@
+import {
+  GITHUB_API_USING_HOST_TYPES,
+  GITLAB_API_USING_HOST_TYPES,
+  PLATFORM_TYPE_GITHUB,
+  PLATFORM_TYPE_GITLAB,
+} from '../../constants/platforms';
 import { logger } from '../../logger';
 import { hasProxy } from '../../proxy';
+import type { HostRule } from '../../types';
 import * as hostRules from '../host-rules';
-import { GotOptions } from './types';
+import type { GotOptions } from './types';
 
-// Apply host rules to requests
+function findMatchingRules(options: GotOptions, url: string): HostRule {
+  const { hostType } = options;
+  let res = hostRules.find({ hostType, url });
+
+  // Fallback to `github` hostType
+  if (
+    GITHUB_API_USING_HOST_TYPES.includes(hostType) &&
+    hostType !== PLATFORM_TYPE_GITHUB
+  ) {
+    res = {
+      ...hostRules.find({
+        hostType: PLATFORM_TYPE_GITHUB,
+        url,
+      }),
+      ...res,
+    };
+  }
+
+  // Fallback to `gitlab` hostType
+  if (
+    GITLAB_API_USING_HOST_TYPES.includes(hostType) &&
+    hostType !== PLATFORM_TYPE_GITLAB
+  ) {
+    res = {
+      ...hostRules.find({
+        hostType: PLATFORM_TYPE_GITLAB,
+        url,
+      }),
+      ...res,
+    };
+  }
 
+  return res;
+}
+
+// Apply host rules to requests
 export function applyHostRules(url: string, inOptions: GotOptions): GotOptions {
   const options: GotOptions = { ...inOptions };
-  const foundRules =
-    hostRules.find({
-      hostType: options.hostType,
-      url,
-    }) || /* istanbul ignore next: can only happen in tests */ {};
+  const foundRules = findMatchingRules(options, url);
   const { username, password, token, enabled, authType } = foundRules;
   if (options.noAuth) {
     logger.trace({ url }, `Authorization disabled`);

lib/util/http/index.spec.ts

@@ -208,4 +208,11 @@ describe('util/http/index', () => {
     expect(bar).toBeTrue();
     expect(baz).toBeTrue();
   });
+
+  it('getBuffer', async () => {
+    httpMock.scope(baseUrl).get('/').reply(200, Buffer.from('test'));
+    const res = await http.getBuffer('http://renovate.com');
+    expect(res.body).toBeInstanceOf(Buffer);
+    expect(res.body.toString('utf-8')).toEqual('test');
+  });
 });

lib/util/http/index.ts

@@ -53,12 +53,16 @@ export interface HttpResponse<T = string> {
   authorization?: boolean;
 }
 
-function cloneResponse<T>(response: any): HttpResponse<T> {
+function cloneResponse<T extends Buffer | string | any>(
+  response: HttpResponse<T>
+): HttpResponse<T> {
+  const { body, statusCode, headers } = response;
   // clone body and headers so that the cached result doesn't get accidentally mutated
+  // Don't use json clone for buffers
   return {
-    statusCode: response.statusCode,
-    body: clone<T>(response.body),
-    headers: clone(response.headers),
+    statusCode,
+    body: body instanceof Buffer ? (body.slice() as T) : clone<T>(body),
+    headers: clone(headers),
     authorization: !!response.authorization,
   };
 }
@@ -95,7 +99,8 @@ async function gotRoutine<T>(
   // Cheat the TS compiler using `as` to pick a specific overload.
   // Otherwise it doesn't typecheck.
   const resp = await got<T>(url, options as GotJSONOptions);
-  const duration = resp.timings.phases.total || 0;
+  const duration =
+    resp.timings.phases.total || /* istanbul ignore next: can't be tested */ 0;
 
   const httpRequests = memCache.get('http-requests') || [];
   httpRequests.push({ ...requestStats, duration });
@@ -166,6 +171,7 @@ export class Http<GetOptions = HttpOptions, PostOptions = HttpPostOptions> {
       resPromise = memCache.get(cacheKey);
     }
 
+    // istanbul ignore else: no cache tests
     if (!resPromise) {
       const startTime = Date.now();
       const queueTask = (): Promise<Response<T>> => {