From 58665d3e56437e612731fd86b18e7fe348280a94 Mon Sep 17 00:00:00 2001 From: Jasper De Moor Date: Tue, 8 May 2018 11:11:10 +0200 Subject: [PATCH 1/4] split cache into subdirs --- src/FSCache.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/FSCache.js b/src/FSCache.js index 8043b43e66f..db5a2f9575a 100644 --- a/src/FSCache.js +++ b/src/FSCache.js @@ -22,14 +22,14 @@ class FSCache { ); } - async ensureDirExists() { - await fs.mkdirp(this.dir); + async ensureDirExists(dir = this.dir) { + await fs.mkdirp(dir); this.dirExists = true; } getCacheFile(filename) { let hash = md5(this.optionsHash + filename); - return path.join(this.dir, hash + '.json'); + return path.join(this.dir, hash.substring(0, 2), hash + '.json'); } async getLastModified(filename) { @@ -56,9 +56,10 @@ class FSCache { async write(filename, data) { try { - await this.ensureDirExists(); + let cacheFile = this.getCacheFile(filename); + await this.ensureDirExists(path.dirname(cacheFile)); await this.writeDepMtimes(data); - await fs.writeFile(this.getCacheFile(filename), JSON.stringify(data)); + await fs.writeFile(cacheFile, JSON.stringify(data)); this.invalidated.delete(filename); } catch (err) { logger.error(`Error writing to cache: ${err.message}`); From 9f9e8318657a90ef7d3c0dce1a870b6777ed1d85 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 21 Jul 2018 19:39:53 -0700 Subject: [PATCH 2/4] Pre-create sub-directories --- src/FSCache.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/FSCache.js b/src/FSCache.js index db5a2f9575a..f62cbc56bbc 100644 --- a/src/FSCache.js +++ b/src/FSCache.js @@ -22,14 +22,25 @@ class FSCache { ); } - async ensureDirExists(dir = this.dir) { - await fs.mkdirp(dir); + async ensureDirExists() { + if (this.dirExists) { + return; + } + + await fs.mkdirp(this.dir); + + // Create sub-directories for every possible hex value + // This speeds up large caches on many file systems since there are fewer files in a single directory. + for (let i = 0; i < 256; i++) { + await fs.mkdirp(path.join(this.dir, i.toString(16).padStart(2, '0'))); + } + this.dirExists = true; } getCacheFile(filename) { let hash = md5(this.optionsHash + filename); - return path.join(this.dir, hash.substring(0, 2), hash + '.json'); + return path.join(this.dir, hash.slice(0, 2), hash.slice(2) + '.json'); } async getLastModified(filename) { @@ -56,8 +67,8 @@ class FSCache { async write(filename, data) { try { + await this.ensureDirExists(); let cacheFile = this.getCacheFile(filename); - await this.ensureDirExists(path.dirname(cacheFile)); await this.writeDepMtimes(data); await fs.writeFile(cacheFile, JSON.stringify(data)); this.invalidated.delete(filename); From 1118a0c38dd8598749817e60792505a92ae12098 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 21 Jul 2018 19:41:56 -0700 Subject: [PATCH 3/4] Fix node 6 --- src/FSCache.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/FSCache.js b/src/FSCache.js index f62cbc56bbc..067558c7b9d 100644 --- a/src/FSCache.js +++ b/src/FSCache.js @@ -32,7 +32,7 @@ class FSCache { // Create sub-directories for every possible hex value // This speeds up large caches on many file systems since there are fewer files in a single directory. for (let i = 0; i < 256; i++) { - await fs.mkdirp(path.join(this.dir, i.toString(16).padStart(2, '0'))); + await fs.mkdirp(path.join(this.dir, ('00' + i.toString(16)).slice(-2))); } this.dirExists = true; From 76690edd6efb5f4ace5b0348d2bf039c032394a9 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 21 Jul 2018 19:43:05 -0700 Subject: [PATCH 4/4] Clean up --- src/FSCache.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/FSCache.js b/src/FSCache.js index 067558c7b9d..fee757bfcec 100644 --- a/src/FSCache.js +++ b/src/FSCache.js @@ -68,9 +68,8 @@ class FSCache { async write(filename, data) { try { await this.ensureDirExists(); - let cacheFile = this.getCacheFile(filename); await this.writeDepMtimes(data); - await fs.writeFile(cacheFile, JSON.stringify(data)); + await fs.writeFile(this.getCacheFile(filename), JSON.stringify(data)); this.invalidated.delete(filename); } catch (err) { logger.error(`Error writing to cache: ${err.message}`);