Skip to content

Commit

Permalink
feat: Add hash of .cargo/config.toml to key (#149)
Browse files Browse the repository at this point in the history 
Also:
 - Add and use `sort_and_uniq` to make sure `globFile` resulting
   and `keyFiles` does not contain duplicates.
 - Only returns regular file in function `globFile`

Signed-off-by: Jiahao XU <Jiahao_XU@outlook.com>
  • Loading branch information
@NobodyXu
NobodyXu committed Jun 18, 2023
1 parent 3d40001 commit 715970f
Show file tree
Hide file tree
Showing 3 changed files with 82 additions and 39 deletions.
39 changes: 26 additions & 13 deletions dist/restore/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64194,15 +64194,13 @@ class CacheConfig {
workspaces.push(new Workspace(root, target));
}
self.workspaces = workspaces;
let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml");
let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml");
const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed
hasher = external_crypto_default().createHash("sha1");
for (const workspace of workspaces) {
const root = workspace.root;
keyFiles.push(...(await globFiles(`${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`)));
const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`))
.filter(file => !external_fs_default().statSync(file).isDirectory());
cargo_manifests.sort((a, b) => a.localeCompare(b));
keyFiles.push(...(await globFiles(`${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`)));
const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`));
for (const cargo_manifest of cargo_manifests) {
try {
const content = await promises_default().readFile(cargo_manifest, { encoding: 'utf8' });
Expand Down Expand Up @@ -64233,9 +64231,7 @@ class CacheConfig {
keyFiles.push(cargo_manifest);
}
}
const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`))
.filter(file => !external_fs_default().statSync(file).isDirectory());
cargo_locks.sort((a, b) => a.localeCompare(b));
const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`));
for (const cargo_lock of cargo_locks) {
try {
const content = await promises_default().readFile(cargo_lock, { encoding: 'utf8' });
Expand All @@ -64259,17 +64255,15 @@ class CacheConfig {
}
}
}
keyFiles = keyFiles.filter(file => !external_fs_default().statSync(file).isDirectory());
keyFiles.sort((a, b) => a.localeCompare(b));
keyFiles = sort_and_uniq(keyFiles);
for (const file of keyFiles) {
for await (const chunk of external_fs_default().createReadStream(file)) {
hasher.update(chunk);
}
}
let lockHash = digest(hasher);
keyFiles.push(...parsedKeyFiles);
keyFiles.sort((a, b) => a.localeCompare(b));
self.keyFiles = keyFiles;
self.keyFiles = sort_and_uniq(keyFiles);
key += `-${lockHash}`;
self.cacheKey = key;
self.cachePaths = [config_CARGO_HOME];
Expand Down Expand Up @@ -64372,7 +64366,26 @@ async function globFiles(pattern) {
const globber = await glob.create(pattern, {
followSymbolicLinks: false,
});
return await globber.glob();
// fs.statSync resolve the symbolic link and returns stat for the
// file it pointed to, so isFile would make sure the resolved
// file is actually a regular file.
return (await globber.glob()).filter(file => external_fs_default().statSync(file).isFile());
}
function sort_and_uniq(a) {
return a
.sort((a, b) => a.localeCompare(b))
.reduce((accumulator, currentValue) => {
const len = accumulator.length;
// If accumulator is empty or its last element != currentValue
// Since array is already sorted, elements with the same value
// are grouped together to be continugous in space.
//
// If currentValue != last element, then it must be unique.
if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) {
accumulator.push(currentValue);
}
return accumulator;
}, []);
}
function sort_object(o) {
if (Array.isArray(o)) {
Expand Down
39 changes: 26 additions & 13 deletions dist/save/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64194,15 +64194,13 @@ class CacheConfig {
workspaces.push(new Workspace(root, target));
}
self.workspaces = workspaces;
let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml");
let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml");
const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed
hasher = external_crypto_default().createHash("sha1");
for (const workspace of workspaces) {
const root = workspace.root;
keyFiles.push(...(await globFiles(`${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`)));
const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`))
.filter(file => !external_fs_default().statSync(file).isDirectory());
cargo_manifests.sort((a, b) => a.localeCompare(b));
keyFiles.push(...(await globFiles(`${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`)));
const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`));
for (const cargo_manifest of cargo_manifests) {
try {
const content = await promises_default().readFile(cargo_manifest, { encoding: 'utf8' });
Expand Down Expand Up @@ -64233,9 +64231,7 @@ class CacheConfig {
keyFiles.push(cargo_manifest);
}
}
const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`))
.filter(file => !external_fs_default().statSync(file).isDirectory());
cargo_locks.sort((a, b) => a.localeCompare(b));
const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`));
for (const cargo_lock of cargo_locks) {
try {
const content = await promises_default().readFile(cargo_lock, { encoding: 'utf8' });
Expand All @@ -64259,17 +64255,15 @@ class CacheConfig {
}
}
}
keyFiles = keyFiles.filter(file => !external_fs_default().statSync(file).isDirectory());
keyFiles.sort((a, b) => a.localeCompare(b));
keyFiles = sort_and_uniq(keyFiles);
for (const file of keyFiles) {
for await (const chunk of external_fs_default().createReadStream(file)) {
hasher.update(chunk);
}
}
let lockHash = digest(hasher);
keyFiles.push(...parsedKeyFiles);
keyFiles.sort((a, b) => a.localeCompare(b));
self.keyFiles = keyFiles;
self.keyFiles = sort_and_uniq(keyFiles);
key += `-${lockHash}`;
self.cacheKey = key;
self.cachePaths = [CARGO_HOME];
Expand Down Expand Up @@ -64372,7 +64366,26 @@ async function globFiles(pattern) {
const globber = await glob.create(pattern, {
followSymbolicLinks: false,
});
return await globber.glob();
// fs.statSync resolve the symbolic link and returns stat for the
// file it pointed to, so isFile would make sure the resolved
// file is actually a regular file.
return (await globber.glob()).filter(file => external_fs_default().statSync(file).isFile());
}
function sort_and_uniq(a) {
return a
.sort((a, b) => a.localeCompare(b))
.reduce((accumulator, currentValue) => {
const len = accumulator.length;
// If accumulator is empty or its last element != currentValue
// Since array is already sorted, elements with the same value
// are grouped together to be continugous in space.
//
// If currentValue != last element, then it must be unique.
if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) {
accumulator.push(currentValue);
}
return accumulator;
}, []);
}
function sort_object(o) {
if (Array.isArray(o)) {
Expand Down
43 changes: 30 additions & 13 deletions src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ export class CacheConfig {
}
self.workspaces = workspaces;

let keyFiles = await globFiles("rust-toolchain\nrust-toolchain.toml");
let keyFiles = await globFiles(".cargo/config.toml\nrust-toolchain\nrust-toolchain.toml");
const parsedKeyFiles = []; // keyFiles that are parsed, pre-processed and hashed

hasher = crypto.createHash("sha1");
Expand All @@ -138,13 +138,11 @@ export class CacheConfig {
const root = workspace.root;
keyFiles.push(
...(await globFiles(
`${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`,
`${root}/**/.cargo/config.toml\n${root}/**/rust-toolchain\n${root}/**/rust-toolchain.toml`,
)),
);

const cargo_manifests = (await globFiles(`${root}/**/Cargo.toml`))
.filter(file => !fs.statSync(file).isDirectory());
cargo_manifests.sort((a, b) => a.localeCompare(b));
const cargo_manifests = sort_and_uniq(await globFiles(`${root}/**/Cargo.toml`));

for (const cargo_manifest of cargo_manifests) {
try {
Expand Down Expand Up @@ -182,9 +180,7 @@ export class CacheConfig {
}
}

const cargo_locks = (await globFiles(`${root}/**/Cargo.lock`))
.filter(file => !fs.statSync(file).isDirectory());
cargo_locks.sort((a, b) => a.localeCompare(b));
const cargo_locks = sort_and_uniq(await globFiles(`${root}/**/Cargo.lock`));

for (const cargo_lock of cargo_locks) {
try {
Expand Down Expand Up @@ -212,8 +208,7 @@ export class CacheConfig {
}
}
}
keyFiles = keyFiles.filter(file => !fs.statSync(file).isDirectory());
keyFiles.sort((a, b) => a.localeCompare(b));
keyFiles = sort_and_uniq(keyFiles);

for (const file of keyFiles) {
for await (const chunk of fs.createReadStream(file)) {
Expand All @@ -224,8 +219,7 @@ export class CacheConfig {
let lockHash = digest(hasher);

keyFiles.push(...parsedKeyFiles);
keyFiles.sort((a, b) => a.localeCompare(b));
self.keyFiles = keyFiles;
self.keyFiles = sort_and_uniq(keyFiles);

key += `-${lockHash}`;
self.cacheKey = key;
Expand Down Expand Up @@ -348,7 +342,30 @@ async function globFiles(pattern: string): Promise<string[]> {
const globber = await glob.create(pattern, {
followSymbolicLinks: false,
});
return await globber.glob();
// fs.statSync resolve the symbolic link and returns stat for the
// file it pointed to, so isFile would make sure the resolved
// file is actually a regular file.
return (await globber.glob()).filter(file => fs.statSync(file).isFile());
}

function sort_and_uniq(a: string[]) {
return a
.sort((a, b) => a.localeCompare(b))
.reduce(
(accumulator: string[], currentValue: string) => {
const len = accumulator.length;
// If accumulator is empty or its last element != currentValue
// Since array is already sorted, elements with the same value
// are grouped together to be continugous in space.
//
// If currentValue != last element, then it must be unique.
if (len == 0 || accumulator[len - 1].localeCompare(currentValue) != 0) {
accumulator.push(currentValue);
}
return accumulator;
},
[]
);
}

function sort_object(o: any): any {
Expand Down

0 comments on commit 715970f

Please sign in to comment.