From 6579769dff97f95927e6ca7275b757faa6affd87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20=C3=85hsberg?= Date: Thu, 22 Oct 2020 11:17:25 +0200 Subject: [PATCH 1/3] Add support for reading files from Bitbucket Server. --- .changeset/3030.md | 5 + packages/backend-common/package.json | 2 +- .../src/reading/BitbucketUrlReader.test.ts | 356 ++++++++++++------ .../src/reading/BitbucketUrlReader.ts | 326 +++++++++++----- yarn.lock | 7 + 5 files changed, 478 insertions(+), 218 deletions(-) create mode 100644 .changeset/3030.md diff --git a/.changeset/3030.md b/.changeset/3030.md new file mode 100644 index 0000000000000..89730dc14c796 --- /dev/null +++ b/.changeset/3030.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': minor +--- + +Add the ability to import components from Bitbucket Server to the service catalog diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 5a2357f74e5f7..875163dd96614 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -41,7 +41,7 @@ "express": "^4.17.1", "express-prom-bundle": "^6.1.0", "express-promise-router": "^3.0.3", - "git-url-parse": "^11.3.0", + "git-url-parse": "^11.4.0", "helmet": "^4.0.0", "knex": "^0.21.1", "lodash": "^4.17.15", diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index bd7c43d88709f..39a4d0db26364 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -14,139 +14,253 @@ * limitations under the License. */ -import { rest } from 'msw'; -import { setupServer } from 'msw/node'; import { ConfigReader } from '@backstage/config'; -import { getVoidLogger } from '../logging'; -import { BitbucketUrlReader } from './BitbucketUrlReader'; -import { msw } from '@backstage/test-utils'; - -const logger = getVoidLogger(); +import { + BitbucketUrlReader, + getApiRequestOptions, + getApiUrl, + getRawRequestOptions, + getRawUrl, + ProviderConfig, + readConfig, +} from './BitbucketUrlReader'; describe('BitbucketUrlReader', () => { - const worker = setupServer(); + describe('getApiRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + host: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getApiRequestOptions(withToken).headers as any).Authorization, + ).toEqual('Bearer A'); + expect( + (getApiRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); + + it('insert basic auth when needed', () => { + const withUsernameAndPassword: ProviderConfig = { + host: '', + apiBaseUrl: '', + username: 'some-user', + appPassword: 'my-secret', + }; + const withoutUsernameAndPassword: ProviderConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getApiRequestOptions(withUsernameAndPassword).headers as any) + .Authorization, + ).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA=='); + expect( + (getApiRequestOptions(withoutUsernameAndPassword).headers as any) + .Authorization, + ).toBeUndefined(); + }); + }); - msw.setupDefaultHandlers(worker); + describe('getRawRequestOptions', () => { + it('inserts a token when needed', () => { + const withToken: ProviderConfig = { + host: '', + apiBaseUrl: '', + token: 'A', + }; + const withoutToken: ProviderConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getRawRequestOptions(withToken).headers as any).Authorization, + ).toEqual('Bearer A'); + expect( + (getRawRequestOptions(withoutToken).headers as any).Authorization, + ).toBeUndefined(); + }); - beforeEach(() => { - worker.use( - rest.get('*', (req, res, ctx) => - res( - ctx.status(200), - ctx.json({ - url: req.url.toString(), - headers: req.headers.getAllHeaders(), - }), + it('insert basic auth when needed', () => { + const withUsernameAndPassword: ProviderConfig = { + host: '', + apiBaseUrl: '', + username: 'some-user', + appPassword: 'my-secret', + }; + const withoutUsernameAndPassword: ProviderConfig = { + host: '', + apiBaseUrl: '', + }; + expect( + (getRawRequestOptions(withUsernameAndPassword).headers as any) + .Authorization, + ).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA=='); + expect( + (getRawRequestOptions(withoutUsernameAndPassword).headers as any) + .Authorization, + ).toBeUndefined(); + }); + }); + + describe('getApiUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + expect(() => getApiUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); + it('happy path for Bitbucket Cloud', () => { + const config: ProviderConfig = { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }; + expect( + getApiUrl( + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config, + ), + ).toEqual( + new URL( + 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', + ), + ); + }); + it('happy path for Bitbucket Server', () => { + const config: ProviderConfig = { + host: 'bitbucket.mycompany.net', + apiBaseUrl: 'https://bitbucket.mycompany.net/rest/api/1.0', + }; + expect( + getApiUrl( + 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml', ), - ), - ); + ); + }); }); - const createConfig = (username?: string, appPassword?: string) => - new ConfigReader( - { - integrations: { - bitbucket: [ - { - host: 'bitbucket.org', - username: username, - appPassword: appPassword, - }, - ], - }, - }, - 'test-config', - ); + describe('getRawUrl', () => { + it('rejects targets that do not look like URLs', () => { + const config: ProviderConfig = { host: '', apiBaseUrl: '' }; + expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); + }); - it.each([ - { - url: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config: createConfig(), - response: expect.objectContaining({ - url: + it('happy path for Bitbucket Cloud', () => { + const config: ProviderConfig = { + host: 'bitbucket.org', + rawBaseUrl: 'https://api.bitbucket.org/2.0', + }; + expect( + getRawUrl( + 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', + config, + ), + ).toEqual( + new URL( 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', - }), - }, - { - url: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config: createConfig('some-user', 'my-secret'), - response: expect.objectContaining({ - headers: expect.objectContaining({ - authorization: 'Basic c29tZS11c2VyOm15LXNlY3JldA==', - }), - }), - }, - { - url: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config: createConfig(), - response: expect.objectContaining({ - headers: expect.not.objectContaining({ - authorization: expect.anything(), - }), - }), - }, - { - url: - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config: createConfig(undefined, 'only-password-provided'), - response: expect.objectContaining({ - headers: expect.not.objectContaining({ - authorization: expect.anything(), - }), - }), - }, - ])('should handle happy path %#', async ({ url, config, response }) => { - const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); + ), + ); + }); + + it('happy path for Bitbucket Server', () => { + const config: ProviderConfig = { + host: 'bitbucket.mycompany.net', + rawBaseUrl: 'https://api.bitbucket.org/2.0', + }; + expect( + getRawUrl( + 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml', + config, + ), + ).toEqual( + new URL( + 'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml', + ), + ); + }); + }); + + describe('readConfig', () => { + function config( + providers: { + host: string; + apiBaseUrl?: string; + token?: string; + username?: string; + password?: string; + }[], + ) { + return ConfigReader.fromConfigs([ + { + context: '', + data: { + integrations: { bitbucket: providers }, + }, + }, + ]); + } + + it('adds a default Bitbucket Cloud entry when missing', () => { + const output = readConfig(config([])); + expect(output).toEqual([ + { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + rawBaseUrl: 'https://api.bitbucket.org/2.0', + }, + ]); + }); + + it('injects the correct Bitbucket Cloud API base URL when missing', () => { + const output = readConfig(config([{ host: 'bitbucket.org' }])); + expect(output).toEqual([ + { + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + rawBaseUrl: 'https://api.bitbucket.org/2.0', + }, + ]); + }); + + it('rejects custom targets with no base URLs', () => { + expect(() => + readConfig(config([{ host: 'bitbucket.mycompany.net' }])), + ).toThrow( + "Bitbucket integration for 'bitbucket.mycompany.net' must configure an explicit apiBaseUrl and rawBaseUrl", + ); + }); - const data = await reader.read(url); - const res = await JSON.parse(data.toString('utf-8')); - expect(res).toEqual(response); + it('rejects funky configs', () => { + expect(() => readConfig(config([{ host: 7 } as any]))).toThrow(/host/); + expect(() => readConfig(config([{ token: 7 } as any]))).toThrow(/token/); + expect(() => + readConfig(config([{ host: 'bitbucket.org', apiBaseUrl: 7 } as any])), + ).toThrow(/apiBaseUrl/); + expect(() => + readConfig(config([{ host: 'bitbucket.org', token: 7 } as any])), + ).toThrow(/token/); + }); }); - it.each([ - { - url: 'https://api.com/a/b/blob/master/path/to/c.yaml', - config: createConfig(), - error: - 'Incorrect url: https://api.com/a/b/blob/master/path/to/c.yaml, Error: Wrong Bitbucket URL or Invalid file path', - }, - { - url: 'com/a/b/blob/master/path/to/c.yaml', - config: createConfig(), - error: - 'Incorrect url: com/a/b/blob/master/path/to/c.yaml, TypeError: Invalid URL: com/a/b/blob/master/path/to/c.yaml', - }, - { - url: '', - config: createConfig('', ''), - error: - "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", - }, - { - url: '', - config: createConfig('only-user-provided', ''), - error: - "Invalid type in config for key 'integrations.bitbucket[0].appPassword' in 'test-config', got empty-string, wanted string", - }, - { - url: '', - config: createConfig('', 'only-password-provided'), - error: - "Invalid type in config for key 'integrations.bitbucket[0].username' in 'test-config', got empty-string, wanted string", - }, - { - url: '', - config: createConfig('only-user-provided', undefined), - error: - "Missing required config value at 'integrations.bitbucket[0].appPassword'", - }, - ])('should handle error path %#', async ({ url, config, error }) => { - await expect(async () => { - const [{ reader }] = BitbucketUrlReader.factory({ config, logger }); - await reader.read(url); - }).rejects.toThrow(error); + describe('implementation', () => { + it('rejects unknown targets', async () => { + const processor = new BitbucketUrlReader({ + host: 'bitbucket.org', + apiBaseUrl: 'https://api.bitbucket.org/2.0', + }); + await expect( + processor.read('https://not.bitbucket.com/apa'), + ).rejects.toThrow( + 'Incorrect URL: https://not.bitbucket.com/apa, Error: Invalid Bitbucket URL or file path', + ); + }); }); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index bf07dc18a6f9a..377ac95a12ae8 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -14,71 +14,262 @@ * limitations under the License. */ -import fetch from 'cross-fetch'; import { Config } from '@backstage/config'; -import { ReaderFactory, UrlReader } from './types'; +import parseGitUri from 'git-url-parse'; +import fetch from 'cross-fetch'; import { NotFoundError } from '../errors'; +import { ReaderFactory, UrlReader } from './types'; + +const DEFAULT_BASE_URL = 'https://api.bitbucket.org/2.0'; -type Options = { - // TODO: added here for future support, but we only allow bitbucket.org for now +/** + * The configuration parameters for a single Bitbucket API provider. + */ +export type ProviderConfig = { + /** + * The host of the target that this matches on, e.g. "bitbucket.com" + */ host: string; - auth?: { - username: string; - appPassword: string; - }; + + /** + * The base URL of the API of this provider, e.g. "https://api.bitbucket.org/2.0", + * with no trailing slash. + * + * May be omitted specifically for Bitbucket Cloud; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + apiBaseUrl?: string; + + /** + * The base URL of the raw fetch endpoint of this provider, e.g. + * "https://api.bitbucket.org/2.0", with no trailing slash. + * + * May be omitted specifically for Bitbucket Cloud; then it will be deduced. + * + * The API will always be preferred if both its base URL and a token are + * present. + */ + rawBaseUrl?: string; + + /** + * The authorization token to use for requests to a Bitbucket Server provider. + * + * See https://confluence.atlassian.com/bitbucketserver/personal-access-tokens-939515499.html + * + * If no token is specified, anonymous access is used. + */ + token?: string; + + /** + * The username to use for requests to Bitbucket Cloud (bitbucket.org). + */ + username?: string; + + /** + * Authentication with Bitbucket Cloud (bitbucket.org) is done using app passwords. + * + * See https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/ + */ + appPassword?: string; }; -function readConfig(config: Config): Options[] { - const optionsArr = Array(); +export function getApiRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + + if (provider.token) { + headers.Authorization = `Bearer ${provider.token}`; + } else if (provider.username && provider.appPassword) { + headers.Authorization = `Basic ${Buffer.from( + `${provider.username}:${provider.appPassword}`, + 'utf8', + ).toString('base64')}`; + } + + return { + headers, + }; +} + +export function getRawRequestOptions(provider: ProviderConfig): RequestInit { + const headers: HeadersInit = {}; + if (provider.token) { + headers.Authorization = `Bearer ${provider.token}`; + } else if (provider.username && provider.appPassword) { + headers.Authorization = `Basic ${Buffer.from( + `${provider.username}:${provider.appPassword}`, + 'utf8', + ).toString('base64')}`; + } + + return { + headers, + }; +} + +// Converts for example +// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml +// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml +export function getApiUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + if ( + !owner || + !name || + (filepathtype !== 'browse' && + filepathtype !== 'raw' && + filepathtype !== 'src') + ) { + throw new Error('Invalid Bitbucket URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + + if (provider.host === 'bitbucket.org') { + if (!ref) { + throw new Error('Invalid Bitbucket URL or file path'); + } + return new URL( + `${provider.apiBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`, + ); + } + return new URL( + `${provider.apiBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +// Converts for example +// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml +// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml +export function getRawUrl(target: string, provider: ProviderConfig): URL { + try { + const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); + + if ( + !owner || + !name || + (filepathtype !== 'browse' && + filepathtype !== 'raw' && + filepathtype !== 'src') + ) { + throw new Error('Invalid Bitbucket URL or file path'); + } + + const pathWithoutSlash = filepath.replace(/^\//, ''); + + if (provider.host === 'bitbucket.org') { + if (!ref) { + throw new Error('Invalid Bitbucket URL or file path'); + } + return new URL( + `${provider.rawBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`, + ); + } + return new URL( + `${provider.rawBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`, + ); + } catch (e) { + throw new Error(`Incorrect URL: ${target}, ${e}`); + } +} + +export function readConfig(config: Config): ProviderConfig[] { + const providers: ProviderConfig[] = []; const providerConfigs = config.getOptionalConfigArray('integrations.bitbucket') ?? []; + // First read all the explicit providers for (const providerConfig of providerConfigs) { const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org'; + let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); + let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); + const token = providerConfig.getOptionalString('token'); + const username = providerConfig.getOptionalString('username'); + const password = providerConfig.getOptionalString('appPassword'); - let auth; - if (providerConfig.has('username')) { - const username = providerConfig.getString('username'); - const appPassword = providerConfig.getString('appPassword'); - auth = { username, appPassword }; + if (apiBaseUrl) { + apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); + } else if (host === 'bitbucket.org') { + apiBaseUrl = DEFAULT_BASE_URL; } - optionsArr.push({ host, auth }); + if (rawBaseUrl) { + rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); + } else if (host === 'bitbucket.org') { + rawBaseUrl = DEFAULT_BASE_URL; + } + + if (!apiBaseUrl && !rawBaseUrl) { + throw new Error( + `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`, + ); + } + if (!token && username && !password) { + throw new Error( + `Bitbucket integration for '${host}' has configured a username but is missing a required password.`, + ); + } + + providers.push({ + host, + apiBaseUrl, + rawBaseUrl, + token, + username, + appPassword: password, + }); } - // As a convenience we always make sure there's at least an unauthenticated - // reader for public bitbucket repos. - if (!optionsArr.some(p => p.host === 'bitbucket.org')) { - optionsArr.push({ host: 'bitbucket.org' }); + // If no explicit bitbucket.org provider was added, put one in the list as + // a convenience + if (!providers.some(p => p.host === 'bitbucket.org')) { + providers.push({ + host: 'bitbucket.org', + apiBaseUrl: DEFAULT_BASE_URL, + rawBaseUrl: DEFAULT_BASE_URL, + }); } - return optionsArr; + return providers; } +/** + * A processor that adds the ability to read files from Bitbucket v1 and v2 APIs, such as + * the one exposed by Bitbucket Cloud itself. + */ export class BitbucketUrlReader implements UrlReader { + private config: ProviderConfig; + static factory: ReaderFactory = ({ config }) => { - return readConfig(config).map(options => { - const reader = new BitbucketUrlReader(options); - const predicate = (url: URL) => url.host === options.host; + return readConfig(config).map(provider => { + const reader = new BitbucketUrlReader(provider); + const predicate = (url: URL) => url.host === provider.host; return { reader, predicate }; }); }; - constructor(private readonly options: Options) { - if (options.host !== 'bitbucket.org') { - throw Error( - `Bitbucket integration currently only supports 'bitbucket.org', tried to use host '${options.host}'`, - ); - } + constructor(config: ProviderConfig) { + this.config = config; } async read(url: string): Promise { - const builtUrl = this.buildRawUrl(url); + const useApi = + this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl); + const bitbucketUrl = useApi + ? getApiUrl(url, this.config) + : getRawUrl(url, this.config); + const options = useApi + ? getApiRequestOptions(this.config) + : getRawRequestOptions(this.config); let response: Response; try { - response = await fetch(builtUrl.toString(), this.getRequestOptions()); + response = await fetch(bitbucketUrl.toString(), options); } catch (e) { throw new Error(`Unable to read ${url}, ${e}`); } @@ -87,76 +278,19 @@ export class BitbucketUrlReader implements UrlReader { return Buffer.from(await response.text()); } - const message = `${url} could not be read as ${builtUrl}, ${response.status} ${response.statusText}`; + const message = `${url} could not be read as ${bitbucketUrl}, ${response.status} ${response.statusText}`; if (response.status === 404) { throw new NotFoundError(message); } throw new Error(message); } - // Converts - // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml - // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml - private buildRawUrl(target: string): URL { - try { - const url = new URL(target); - - const [ - empty, - userOrOrg, - repoName, - srcKeyword, - ref, - ...restOfPath - ] = url.pathname.split('/'); - - if ( - url.hostname !== 'bitbucket.org' || - empty !== '' || - userOrOrg === '' || - repoName === '' || - srcKeyword !== 'src' - ) { - throw new Error('Wrong Bitbucket URL or Invalid file path'); - } - - // transform to api - url.pathname = [ - empty, - '2.0', - 'repositories', - userOrOrg, - repoName, - 'src', - ref, - ...restOfPath, - ].join('/'); - url.hostname = 'api.bitbucket.org'; - url.protocol = 'https'; - - return url; - } catch (e) { - throw new Error(`Incorrect url: ${target}, ${e}`); - } - } - - private getRequestOptions(): RequestInit { - const headers: HeadersInit = {}; - - if (this.options.auth) { - headers.Authorization = `Basic ${Buffer.from( - `${this.options.auth.username}:${this.options.auth.appPassword}`, - 'utf8', - ).toString('base64')}`; - } - - return { - headers, - }; - } - toString() { - const { host, auth } = this.options; - return `bitbucket{host=${host},authed=${Boolean(auth)}}`; + const { host, token, username, appPassword } = this.config; + let authed = Boolean(token); + if (!authed) { + authed = Boolean(username && appPassword); + } + return `bitbucket{host=${host},authed=${authed}}`; } } diff --git a/yarn.lock b/yarn.lock index aeba84153a7a1..c55075145f5cc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12249,6 +12249,13 @@ git-url-parse@^11.3.0: dependencies: git-up "^4.0.0" +git-url-parse@^11.4.0: + version "11.4.0" + resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.0.tgz#f2bb1f2b00f05552540e95a62e31399a639a6aa6" + integrity sha512-KlIa5jvMYLjXMQXkqpFzobsyD/V2K5DRHl5OAf+6oDFPlPLxrGDVQlIdI63c4/Kt6kai4kALENSALlzTGST3GQ== + dependencies: + git-up "^4.0.0" + gitconfiglocal@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz#41d045f3851a5ea88f03f24ca1c6178114464b9b" From 15b57a184814f855043919ec8f20e13cc4f1f85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20=C3=85hsberg?= Date: Fri, 23 Oct 2020 10:27:13 +0200 Subject: [PATCH 2/3] Remove unnecessary rawApi reading --- .../src/reading/BitbucketUrlReader.test.ts | 88 +---------------- .../src/reading/BitbucketUrlReader.ts | 94 ++----------------- 2 files changed, 10 insertions(+), 172 deletions(-) diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts index 39a4d0db26364..210ee16873b3a 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.test.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.test.ts @@ -19,8 +19,6 @@ import { BitbucketUrlReader, getApiRequestOptions, getApiUrl, - getRawRequestOptions, - getRawUrl, ProviderConfig, readConfig, } from './BitbucketUrlReader'; @@ -67,47 +65,6 @@ describe('BitbucketUrlReader', () => { }); }); - describe('getRawRequestOptions', () => { - it('inserts a token when needed', () => { - const withToken: ProviderConfig = { - host: '', - apiBaseUrl: '', - token: 'A', - }; - const withoutToken: ProviderConfig = { - host: '', - apiBaseUrl: '', - }; - expect( - (getRawRequestOptions(withToken).headers as any).Authorization, - ).toEqual('Bearer A'); - expect( - (getRawRequestOptions(withoutToken).headers as any).Authorization, - ).toBeUndefined(); - }); - - it('insert basic auth when needed', () => { - const withUsernameAndPassword: ProviderConfig = { - host: '', - apiBaseUrl: '', - username: 'some-user', - appPassword: 'my-secret', - }; - const withoutUsernameAndPassword: ProviderConfig = { - host: '', - apiBaseUrl: '', - }; - expect( - (getRawRequestOptions(withUsernameAndPassword).headers as any) - .Authorization, - ).toEqual('Basic c29tZS11c2VyOm15LXNlY3JldA=='); - expect( - (getRawRequestOptions(withoutUsernameAndPassword).headers as any) - .Authorization, - ).toBeUndefined(); - }); - }); - describe('getApiUrl', () => { it('rejects targets that do not look like URLs', () => { const config: ProviderConfig = { host: '', apiBaseUrl: '' }; @@ -147,47 +104,6 @@ describe('BitbucketUrlReader', () => { }); }); - describe('getRawUrl', () => { - it('rejects targets that do not look like URLs', () => { - const config: ProviderConfig = { host: '', apiBaseUrl: '' }; - expect(() => getRawUrl('a/b', config)).toThrow(/Incorrect URL: a\/b/); - }); - - it('happy path for Bitbucket Cloud', () => { - const config: ProviderConfig = { - host: 'bitbucket.org', - rawBaseUrl: 'https://api.bitbucket.org/2.0', - }; - expect( - getRawUrl( - 'https://bitbucket.org/org-name/repo-name/src/master/templates/my-template.yaml', - config, - ), - ).toEqual( - new URL( - 'https://api.bitbucket.org/2.0/repositories/org-name/repo-name/src/master/templates/my-template.yaml', - ), - ); - }); - - it('happy path for Bitbucket Server', () => { - const config: ProviderConfig = { - host: 'bitbucket.mycompany.net', - rawBaseUrl: 'https://api.bitbucket.org/2.0', - }; - expect( - getRawUrl( - 'https://bitbucket.mycompany.net/projects/a/repos/b/browse/path/to/c.yaml', - config, - ), - ).toEqual( - new URL( - 'https://bitbucket.mycompany.net/rest/api/1.0/projects/a/repos/b/raw/path/to/c.yaml', - ), - ); - }); - }); - describe('readConfig', () => { function config( providers: { @@ -214,7 +130,6 @@ describe('BitbucketUrlReader', () => { { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0', - rawBaseUrl: 'https://api.bitbucket.org/2.0', }, ]); }); @@ -225,7 +140,6 @@ describe('BitbucketUrlReader', () => { { host: 'bitbucket.org', apiBaseUrl: 'https://api.bitbucket.org/2.0', - rawBaseUrl: 'https://api.bitbucket.org/2.0', }, ]); }); @@ -234,7 +148,7 @@ describe('BitbucketUrlReader', () => { expect(() => readConfig(config([{ host: 'bitbucket.mycompany.net' }])), ).toThrow( - "Bitbucket integration for 'bitbucket.mycompany.net' must configure an explicit apiBaseUrl and rawBaseUrl", + "Bitbucket integration for 'bitbucket.mycompany.net' must configure an explicit apiBaseUrl", ); }); diff --git a/packages/backend-common/src/reading/BitbucketUrlReader.ts b/packages/backend-common/src/reading/BitbucketUrlReader.ts index 377ac95a12ae8..348426fd347ea 100644 --- a/packages/backend-common/src/reading/BitbucketUrlReader.ts +++ b/packages/backend-common/src/reading/BitbucketUrlReader.ts @@ -42,17 +42,6 @@ export type ProviderConfig = { */ apiBaseUrl?: string; - /** - * The base URL of the raw fetch endpoint of this provider, e.g. - * "https://api.bitbucket.org/2.0", with no trailing slash. - * - * May be omitted specifically for Bitbucket Cloud; then it will be deduced. - * - * The API will always be preferred if both its base URL and a token are - * present. - */ - rawBaseUrl?: string; - /** * The authorization token to use for requests to a Bitbucket Server provider. * @@ -92,22 +81,6 @@ export function getApiRequestOptions(provider: ProviderConfig): RequestInit { }; } -export function getRawRequestOptions(provider: ProviderConfig): RequestInit { - const headers: HeadersInit = {}; - if (provider.token) { - headers.Authorization = `Bearer ${provider.token}`; - } else if (provider.username && provider.appPassword) { - headers.Authorization = `Basic ${Buffer.from( - `${provider.username}:${provider.appPassword}`, - 'utf8', - ).toString('base64')}`; - } - - return { - headers, - }; -} - // Converts for example // from: https://bitbucket.org/orgname/reponame/src/master/file.yaml // to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml @@ -142,41 +115,6 @@ export function getApiUrl(target: string, provider: ProviderConfig): URL { } } -// Converts for example -// from: https://bitbucket.org/orgname/reponame/src/master/file.yaml -// to: https://api.bitbucket.org/2.0/repositories/orgname/reponame/src/master/file.yaml -export function getRawUrl(target: string, provider: ProviderConfig): URL { - try { - const { owner, name, ref, filepathtype, filepath } = parseGitUri(target); - - if ( - !owner || - !name || - (filepathtype !== 'browse' && - filepathtype !== 'raw' && - filepathtype !== 'src') - ) { - throw new Error('Invalid Bitbucket URL or file path'); - } - - const pathWithoutSlash = filepath.replace(/^\//, ''); - - if (provider.host === 'bitbucket.org') { - if (!ref) { - throw new Error('Invalid Bitbucket URL or file path'); - } - return new URL( - `${provider.rawBaseUrl}/repositories/${owner}/${name}/src/${ref}/${pathWithoutSlash}`, - ); - } - return new URL( - `${provider.rawBaseUrl}/projects/${owner}/repos/${name}/raw/${pathWithoutSlash}?at=${ref}`, - ); - } catch (e) { - throw new Error(`Incorrect URL: ${target}, ${e}`); - } -} - export function readConfig(config: Config): ProviderConfig[] { const providers: ProviderConfig[] = []; @@ -187,10 +125,9 @@ export function readConfig(config: Config): ProviderConfig[] { for (const providerConfig of providerConfigs) { const host = providerConfig.getOptionalString('host') ?? 'bitbucket.org'; let apiBaseUrl = providerConfig.getOptionalString('apiBaseUrl'); - let rawBaseUrl = providerConfig.getOptionalString('rawBaseUrl'); const token = providerConfig.getOptionalString('token'); const username = providerConfig.getOptionalString('username'); - const password = providerConfig.getOptionalString('appPassword'); + const appPassword = providerConfig.getOptionalString('appPassword'); if (apiBaseUrl) { apiBaseUrl = apiBaseUrl.replace(/\/+$/, ''); @@ -198,30 +135,23 @@ export function readConfig(config: Config): ProviderConfig[] { apiBaseUrl = DEFAULT_BASE_URL; } - if (rawBaseUrl) { - rawBaseUrl = rawBaseUrl.replace(/\/+$/, ''); - } else if (host === 'bitbucket.org') { - rawBaseUrl = DEFAULT_BASE_URL; - } - - if (!apiBaseUrl && !rawBaseUrl) { + if (!apiBaseUrl) { throw new Error( - `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl and rawBaseUrl`, + `Bitbucket integration for '${host}' must configure an explicit apiBaseUrl`, ); } - if (!token && username && !password) { + if (!token && username && !appPassword) { throw new Error( - `Bitbucket integration for '${host}' has configured a username but is missing a required password.`, + `Bitbucket integration for '${host}' has configured a username but is missing a required appPassword.`, ); } providers.push({ host, apiBaseUrl, - rawBaseUrl, token, username, - appPassword: password, + appPassword, }); } @@ -231,7 +161,6 @@ export function readConfig(config: Config): ProviderConfig[] { providers.push({ host: 'bitbucket.org', apiBaseUrl: DEFAULT_BASE_URL, - rawBaseUrl: DEFAULT_BASE_URL, }); } @@ -258,14 +187,9 @@ export class BitbucketUrlReader implements UrlReader { } async read(url: string): Promise { - const useApi = - this.config.apiBaseUrl && (this.config.token || !this.config.rawBaseUrl); - const bitbucketUrl = useApi - ? getApiUrl(url, this.config) - : getRawUrl(url, this.config); - const options = useApi - ? getApiRequestOptions(this.config) - : getRawRequestOptions(this.config); + const bitbucketUrl = getApiUrl(url, this.config); + + const options = getApiRequestOptions(this.config); let response: Response; try { From 834f602eafa98aad264d5d08e0947a8035be720d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mathias=20=C3=85hsberg?= Date: Sat, 24 Oct 2020 16:19:54 +0200 Subject: [PATCH 3/3] Upgrade git-url-parse to v11.4.0 --- plugins/catalog-backend/package.json | 2 +- plugins/scaffolder-backend/package.json | 2 +- plugins/techdocs-backend/package.json | 2 +- yarn.lock | 7 ------- 4 files changed, 3 insertions(+), 10 deletions(-) diff --git a/plugins/catalog-backend/package.json b/plugins/catalog-backend/package.json index ecc154bc5a55c..7d41da28886f4 100644 --- a/plugins/catalog-backend/package.json +++ b/plugins/catalog-backend/package.json @@ -31,7 +31,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.3.0", + "git-url-parse": "^11.4.0", "knex": "^0.21.1", "ldapjs": "^2.2.0", "lodash": "^4.17.15", diff --git a/plugins/scaffolder-backend/package.json b/plugins/scaffolder-backend/package.json index 66d6457333c76..d0fb832538a16 100644 --- a/plugins/scaffolder-backend/package.json +++ b/plugins/scaffolder-backend/package.json @@ -36,7 +36,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.0", - "git-url-parse": "^11.3.0", + "git-url-parse": "^11.4.0", "globby": "^11.0.0", "helmet": "^4.0.0", "jsonschema": "^1.2.6", diff --git a/plugins/techdocs-backend/package.json b/plugins/techdocs-backend/package.json index 60c7c8f084f97..e3878e95b5bba 100644 --- a/plugins/techdocs-backend/package.json +++ b/plugins/techdocs-backend/package.json @@ -30,7 +30,7 @@ "express": "^4.17.1", "express-promise-router": "^3.0.3", "fs-extra": "^9.0.1", - "git-url-parse": "^11.3.0", + "git-url-parse": "^11.4.0", "knex": "^0.21.1", "nodegit": "^0.27.0", "cross-fetch": "^3.0.6", diff --git a/yarn.lock b/yarn.lock index c55075145f5cc..f44249adb14ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12242,13 +12242,6 @@ git-url-parse@^11.1.2: dependencies: git-up "^4.0.0" -git-url-parse@^11.3.0: - version "11.3.0" - resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.3.0.tgz#1515b4574c4eb2efda7d25cc50b29ce8beaefaae" - integrity sha512-i3XNa8IKmqnUqWBcdWBjOcnyZYfN3C1WRvnKI6ouFWwsXCZEnlgbwbm55ZpJ3OJMhfEP/ryFhqW8bBhej3C5Ug== - dependencies: - git-up "^4.0.0" - git-url-parse@^11.4.0: version "11.4.0" resolved "https://registry.npmjs.org/git-url-parse/-/git-url-parse-11.4.0.tgz#f2bb1f2b00f05552540e95a62e31399a639a6aa6"