Skip to content

Commit

Permalink
refactor: simplify crate datasource
Browse files Browse the repository at this point in the history
  • Loading branch information
@rarkins
rarkins committed Jun 23, 2020
1 parent a905012 commit e4e76f8
Show file tree
Hide file tree
Showing 4 changed files with 19 additions and 249 deletions.
1 change: 0 additions & 1 deletion lib/datasource/crate/__fixtures__/invalid_crate_data
View file

This file was deleted.

181 changes: 0 additions & 181 deletions lib/datasource/crate/__snapshots__/index.spec.ts.snap
View file
Expand Up @@ -501,101 +501,6 @@ Array [
]
`;

exports[`datasource/crate getReleases returns null for invalid crate data 1`] = `
Array [
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
]
`;

exports[`datasource/crate getReleases returns null for missing fields 1`] = `
Array [
Object {
Expand Down Expand Up @@ -678,92 +583,6 @@ Array [
]
`;

exports[`datasource/crate getReleases returns null if crate name is invalid 1`] = `
Array [
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst",
},
Object {
"headers": Object {
"accept-encoding": "gzip, deflate",
"host": "raw.githubusercontent.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name",
},
]
`;

exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: external-host-error]`;

exports[`datasource/crate getReleases throws for 5xx 2`] = `
Expand Down
22 changes: 0 additions & 22 deletions lib/datasource/crate/index.spec.ts
View file
Expand Up @@ -9,10 +9,6 @@ const res2 = fs.readFileSync(
'lib/datasource/crate/__fixtures__/amethyst',
'utf8'
);
const res3 = fs.readFileSync(
'lib/datasource/crate/__fixtures__/invalid_crate_data',
'utf8'
);

const baseUrl =
'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
Expand Down Expand Up @@ -100,23 +96,5 @@ describe('datasource/crate', () => {
expect(res).toBeDefined();
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('returns null if crate name is invalid', async () => {
httpMock.scope(baseUrl).get('/in/va/invalid-crate-name').reply(200, res2);
const res = await getPkgReleases({
datasource,
depName: 'invalid-crate-name',
});
expect(res).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('returns null for invalid crate data', async () => {
httpMock.scope(baseUrl).get('/so/me/some_crate').reply(200, res3);
const res = await getPkgReleases({
datasource,
depName: 'some_crate',
});
expect(res).toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
});
});
64 changes: 19 additions & 45 deletions lib/datasource/crate/index.ts
View file
Expand Up @@ -40,54 +40,28 @@ export async function getReleases({
'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/';
const crateUrl = baseUrl + path;
try {
let res: any = await http.get(crateUrl);
if (!res || !res.body) {
logger.warn(
{ dependency: lookupName },
`Received invalid crate data from ${crateUrl}`
);
return null;
}
res = res.body;
res = res.split('\n');
res = res.map((line) => line.trim()).filter((line) => line.length !== 0);
if (res.length === 0) {
logger.warn(
{ dependency: lookupName },
`Received empty list from ${crateUrl}`
);
return null;
}
// Filter empty lines (takes care of trailing \n)
// eslint-disable-next-line @typescript-eslint/unbound-method
res = res.map(JSON.parse);
if (res[0].name !== lookupName) {
logger.warn(
{ dependency: lookupName },
`Received invalid crate name from ${crateUrl}`
);
return null;
}
if (!res[0].vers) {
logger.warn(
{ dependency: lookupName },
`Recieved invalid data (vers field doesn't exist) from ${crateUrl}`
);
return null;
}
const lines = (await http.get(crateUrl)).body
.split('\n') // break into lines
.map((line) => line.trim()) // remove whitespace
.filter((line) => line.length !== 0) // remove empty lines
.map((line) => JSON.parse(line)); // parse
const result: ReleaseResult = {
releases: [],
};
result.releases = res.map((version: { vers: string; yanked: boolean }) => {
const release: Release = {
version: version.vers,
};
if (version.yanked) {
release.isDeprecated = true;
}
return release;
});

result.releases = lines
.map((version: { vers: string; yanked: boolean }) => {
const release: Release = {
version: version.vers,
};
if (version.yanked) {
release.isDeprecated = true;
}
return release;
})
.filter((release) => release.version);
if (!result.releases.length) {
return null;
}
const cacheMinutes = 10;
await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes);
return result;
Expand Down

0 comments on commit e4e76f8

Please sign in to comment.