From f99e40779126448fc29e1d36fb8b041d3ec45932 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 26 Oct 2021 18:41:21 -0400
Subject: [PATCH 01/17] Add large blob methods to cache

---
 packages/core/cache/src/FSCache.js       | 12 ++++++++++++
 packages/core/cache/src/LMDBCache.js     | 22 +++++++++++++++++++---
 packages/core/cache/src/types.js         |  3 +++
 packages/core/core/src/resolveOptions.js |  2 +-
 4 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/packages/core/cache/src/FSCache.js b/packages/core/cache/src/FSCache.js
index ef7ba6da5db..86132317917 100644
--- a/packages/core/cache/src/FSCache.js
+++ b/packages/core/cache/src/FSCache.js
@@ -77,6 +77,18 @@ export class FSCache implements Cache {
     }
   }
 
+  hasLargeBlob(key: string): Promise<boolean> {
+    return this.has(key);
+  }
+
+  getLargeBlob(key: string): Promise<Buffer> {
+    return this.getBlob(key);
+  }
+
+  setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
+    return this.setBlob(key, contents);
+  }
+
   async get<T>(key: string): Promise<?T> {
     try {
       let data = await this.fs.readFile(this._getCachePath(key));
diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 747ad49ad73..05ae0803b89 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -1,8 +1,10 @@
 // @flow strict-local
 import type {Readable} from 'stream';
 import type {FilePath} from '@parcel/types';
+import type {FileSystem} from '@parcel/fs';
 import type {Cache} from './types';
 
+import path from 'path';
 import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
 import {blobToStream, bufferStream} from '@parcel/utils';
 // flowlint-next-line untyped-import:off
@@ -11,11 +13,13 @@ import packageJson from '../package.json';
 import lmdb from 'lmdb-store';
 
 export class LMDBCache implements Cache {
+  fs: FileSystem;
   dir: FilePath;
   // $FlowFixMe
   store: any;
 
-  constructor(cacheDir: FilePath) {
+  constructor(fs: FileSystem, cacheDir: FilePath) {
+    this.fs = fs;
     this.dir = cacheDir;
 
     this.store = lmdb.open(cacheDir, {
@@ -35,8 +39,8 @@ export class LMDBCache implements Cache {
     };
   }
 
-  static deserialize(opts: {|dir: FilePath|}): LMDBCache {
-    return new LMDBCache(opts.dir);
+  static deserialize(opts: {|fs: FileSystem, dir: FilePath|}): LMDBCache {
+    return new LMDBCache(opts.fs, opts.dir);
   }
 
   has(key: string): Promise<boolean> {
@@ -79,6 +83,18 @@ export class LMDBCache implements Cache {
   getBuffer(key: string): Promise<?Buffer> {
     return Promise.resolve(this.store.get(key));
   }
+
+  hasLargeBlob(key: string): Promise<boolean> {
+    return this.fs.exists(path.join(this.dir, key));
+  }
+
+  getLargeBlob(key: string): Promise<Buffer> {
+    return this.fs.readFile(path.join(this.dir, key));
+  }
+
+  async setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
+    await this.fs.writeFile(path.join(this.dir, key), contents);
+  }
 }
 
 registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);
diff --git a/packages/core/cache/src/types.js b/packages/core/cache/src/types.js
index 5f591388048..1211daf844b 100644
--- a/packages/core/cache/src/types.js
+++ b/packages/core/cache/src/types.js
@@ -10,5 +10,8 @@ export interface Cache {
   setStream(key: string, stream: Readable): Promise<void>;
   getBlob(key: string): Promise<Buffer>;
   setBlob(key: string, contents: Buffer | string): Promise<void>;
+  hasLargeBlob(key: string): Promise<boolean>;
+  getLargeBlob(key: string): Promise<Buffer>;
+  setLargeBlob(key: string, contents: Buffer | string): Promise<void>;
   getBuffer(key: string): Promise<?Buffer>;
 }
diff --git a/packages/core/core/src/resolveOptions.js b/packages/core/core/src/resolveOptions.js
index 1ce2e208571..5ca429b7215 100644
--- a/packages/core/core/src/resolveOptions.js
+++ b/packages/core/core/src/resolveOptions.js
@@ -74,7 +74,7 @@ export default async function resolveOptions(
   let cache =
     initialOptions.cache ??
     (outputFS instanceof NodeFS
-      ? new LMDBCache(cacheDir)
+      ? new LMDBCache(outputFS, cacheDir)
       : new FSCache(outputFS, cacheDir));
 
   let mode = initialOptions.mode ?? 'development';

From f6c22d7f55346fab6b86c20a6c749e8fcd4c2a82 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 26 Oct 2021 18:42:52 -0400
Subject: [PATCH 02/17] Cache request graph as large blob

---
 packages/core/core/src/RequestTracker.js | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js
index 755459b48be..bb259aee7f7 100644
--- a/packages/core/core/src/RequestTracker.js
+++ b/packages/core/core/src/RequestTracker.js
@@ -24,6 +24,7 @@ import {
 } from '@parcel/utils';
 import {hashString} from '@parcel/hash';
 import {ContentGraph} from '@parcel/graph';
+import {deserialize, serialize} from './serializer';
 import {assertSignalNotAborted, hashFromOption} from './utils';
 import {
   type ProjectPath,
@@ -1039,7 +1040,9 @@ export default class RequestTracker {
       }
     }
 
-    promises.push(this.options.cache.set(requestGraphKey, this.graph));
+    promises.push(
+      this.options.cache.setLargeBlob(requestGraphKey, serialize(this.graph)),
+    );
 
     let opts = getWatcherOptions(this.options);
     let snapshotPath = path.join(this.options.cacheDir, snapshotKey + '.txt');
@@ -1083,9 +1086,10 @@ async function loadRequestGraph(options): Async<RequestGraph> {
 
   let cacheKey = getCacheKey(options);
   let requestGraphKey = hashString(`${cacheKey}:requestGraph`);
-  let requestGraph = await options.cache.get<RequestGraph>(requestGraphKey);
-
-  if (requestGraph) {
+  if (await options.cache.hasLargeBlob(requestGraphKey)) {
+    let requestGraph: RequestGraph = deserialize(
+      await options.cache.getLargeBlob(requestGraphKey),
+    );
     let opts = getWatcherOptions(options);
     let snapshotKey = hashString(`${cacheKey}:snapshot`);
     let snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');

From 8f0a103ac2713ae9884e2b802337cb20b99a7373 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Wed, 27 Oct 2021 12:33:18 -0400
Subject: [PATCH 03/17] Fix LMDBCache serialization

---
 packages/core/cache/src/LMDBCache.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 05ae0803b89..bdb298669b8 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -33,8 +33,9 @@ export class LMDBCache implements Cache {
     return Promise.resolve();
   }
 
-  serialize(): {|dir: FilePath|} {
+  serialize(): {|fs: FileSystem, dir: FilePath|} {
     return {
+      fs: this.fs,
       dir: this.dir,
     };
   }

From aec63a85ebf1c46a1e3a01c0d962cd7a5144b46c Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Wed, 27 Oct 2021 14:53:52 -0400
Subject: [PATCH 04/17] Prepare LMDBCache fs for serialization

---
 packages/core/cache/src/LMDBCache.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index bdb298669b8..2f9fe363e6b 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -5,7 +5,12 @@ import type {FileSystem} from '@parcel/fs';
 import type {Cache} from './types';
 
 import path from 'path';
-import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
+import {
+  serialize,
+  deserialize,
+  prepareForSerialization,
+  registerSerializableClass,
+} from '@parcel/core';
 import {blobToStream, bufferStream} from '@parcel/utils';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
@@ -35,7 +40,7 @@ export class LMDBCache implements Cache {
 
   serialize(): {|fs: FileSystem, dir: FilePath|} {
     return {
-      fs: this.fs,
+      fs: prepareForSerialization(this.fs),
       dir: this.dir,
     };
   }

From 6681eefad9ba58acc4b40a8f51c6039d27bf3b96 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 2 Nov 2021 14:01:31 -0400
Subject: [PATCH 05/17] Cache graph request results as large blobs

---
 packages/core/core/src/RequestTracker.js | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/packages/core/core/src/RequestTracker.js b/packages/core/core/src/RequestTracker.js
index bb259aee7f7..51d70a6bd1e 100644
--- a/packages/core/core/src/RequestTracker.js
+++ b/packages/core/core/src/RequestTracker.js
@@ -840,10 +840,11 @@ export default class RequestTracker {
       let result: T = (node.value.result: any);
       return result;
     } else if (node.value.resultCacheKey != null && ifMatch == null) {
-      let cachedResult: T = (nullthrows(
-        await this.options.cache.get(node.value.resultCacheKey),
-        // $FlowFixMe
-      ): any);
+      let key = node.value.resultCacheKey;
+      invariant(this.options.cache.hasLargeBlob(key));
+      let cachedResult: T = deserialize(
+        await this.options.cache.getLargeBlob(key),
+      );
       node.value.result = cachedResult;
       return cachedResult;
     }
@@ -1034,7 +1035,10 @@ export default class RequestTracker {
       let resultCacheKey = node.value.resultCacheKey;
       if (resultCacheKey != null && node.value.result != null) {
         promises.push(
-          this.options.cache.set(resultCacheKey, node.value.result),
+          this.options.cache.setLargeBlob(
+            resultCacheKey,
+            serialize(node.value.result),
+          ),
         );
         delete node.value.result;
       }

From 6c88b5d7c838f3a8e2cf0eadc28810a3073d6b28 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 2 Nov 2021 16:58:39 -0400
Subject: [PATCH 06/17] Revert configurable LMDBCache FS

`LMDBCache` is only used when the FS is `NodeFS` anyway.
---
 packages/core/cache/src/LMDBCache.js     | 22 ++++++++--------------
 packages/core/core/src/resolveOptions.js |  2 +-
 2 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 2f9fe363e6b..a9ac101a9de 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -1,16 +1,11 @@
 // @flow strict-local
 import type {Readable} from 'stream';
 import type {FilePath} from '@parcel/types';
-import type {FileSystem} from '@parcel/fs';
 import type {Cache} from './types';
 
 import path from 'path';
-import {
-  serialize,
-  deserialize,
-  prepareForSerialization,
-  registerSerializableClass,
-} from '@parcel/core';
+import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
+import {NodeFS} from '@parcel/fs';
 import {blobToStream, bufferStream} from '@parcel/utils';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
@@ -18,13 +13,13 @@ import packageJson from '../package.json';
 import lmdb from 'lmdb-store';
 
 export class LMDBCache implements Cache {
-  fs: FileSystem;
+  fs: NodeFS;
   dir: FilePath;
   // $FlowFixMe
   store: any;
 
-  constructor(fs: FileSystem, cacheDir: FilePath) {
-    this.fs = fs;
+  constructor(cacheDir: FilePath) {
+    this.fs = new NodeFS();
     this.dir = cacheDir;
 
     this.store = lmdb.open(cacheDir, {
@@ -38,15 +33,14 @@ export class LMDBCache implements Cache {
     return Promise.resolve();
   }
 
-  serialize(): {|fs: FileSystem, dir: FilePath|} {
+  serialize(): {|dir: FilePath|} {
     return {
-      fs: prepareForSerialization(this.fs),
       dir: this.dir,
     };
   }
 
-  static deserialize(opts: {|fs: FileSystem, dir: FilePath|}): LMDBCache {
-    return new LMDBCache(opts.fs, opts.dir);
+  static deserialize(opts: {|dir: FilePath|}): LMDBCache {
+    return new LMDBCache(opts.dir);
   }
 
   has(key: string): Promise<boolean> {
diff --git a/packages/core/core/src/resolveOptions.js b/packages/core/core/src/resolveOptions.js
index 5ca429b7215..1ce2e208571 100644
--- a/packages/core/core/src/resolveOptions.js
+++ b/packages/core/core/src/resolveOptions.js
@@ -74,7 +74,7 @@ export default async function resolveOptions(
   let cache =
     initialOptions.cache ??
     (outputFS instanceof NodeFS
-      ? new LMDBCache(outputFS, cacheDir)
+      ? new LMDBCache(cacheDir)
       : new FSCache(outputFS, cacheDir));
 
   let mode = initialOptions.mode ?? 'development';

From 74b4dd49c4eda38e306615f15ac171a7c4d7bdac Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Wed, 3 Nov 2021 13:31:58 -0400
Subject: [PATCH 07/17] Fall back to FS for large blobs in LMDBCache

---
 packages/core/cache/src/LMDBCache.js | 55 ++++++++++++++++++----------
 1 file changed, 35 insertions(+), 20 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index a9ac101a9de..2f8c988fa64 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -44,44 +44,46 @@ export class LMDBCache implements Cache {
   }
 
   has(key: string): Promise<boolean> {
-    return Promise.resolve(this.store.get(key) != null);
+    if (this.store.get(key) != null) return Promise.resolve(true);
+    return this.hasLargeBlob(key);
   }
 
-  get<T>(key: string): Promise<?T> {
-    let data = this.store.get(key);
-    if (data == null) {
-      return Promise.resolve(null);
-    }
-
-    return Promise.resolve(deserialize(data));
+  async get<T>(key: string): Promise<?T> {
+    let data = await this.getBuffer(key);
+    return data == null ? null : deserialize(data);
   }
 
   async set(key: string, value: mixed): Promise<void> {
-    await this.store.put(key, serialize(value));
+    await this.setBlob(key, serialize(value));
   }
 
   getStream(key: string): Readable {
-    return blobToStream(this.store.get(key));
+    return blobToStream(
+      this.store.get(key) ?? this.fs.readFileSync(path.join(this.dir, key)),
+    );
   }
 
   async setStream(key: string, stream: Readable): Promise<void> {
-    let buf = await bufferStream(stream);
-    await this.store.put(key, buf);
+    await this.setBlob(key, await bufferStream(stream));
   }
 
-  getBlob(key: string): Promise<Buffer> {
-    let buffer = this.store.get(key);
-    return buffer != null
-      ? Promise.resolve(buffer)
-      : Promise.reject(new Error(`Key ${key} not found in cache`));
+  async getBlob(key: string): Promise<Buffer> {
+    let buffer = await this.getBuffer(key);
+    if (buffer == null) throw new Error(`Key ${key} not found in cache`);
+    return buffer;
   }
 
   async setBlob(key: string, contents: Buffer | string): Promise<void> {
-    await this.store.put(key, contents);
+    if (isLargeBlob(contents)) await this.setLargeBlob(key, contents);
+    else await this.store.put(key, contents);
   }
 
-  getBuffer(key: string): Promise<?Buffer> {
-    return Promise.resolve(this.store.get(key));
+  async getBuffer(key: string): Promise<?Buffer> {
+    let buffer = this.store.get(key);
+    if (buffer == null && (await this.hasLargeBlob(key))) {
+      buffer = await this.getLargeBlob(key);
+    }
+    return buffer;
   }
 
   hasLargeBlob(key: string): Promise<boolean> {
@@ -97,4 +99,17 @@ export class LMDBCache implements Cache {
   }
 }
 
+// lmbd-store decodes cached binary data into a Node Buffer
+// via `Nan::NewBuffer`, which enforces a max size of ~1GB.
+// We subtract 9 bytes to account for any compression heaader
+// added by lmbd-store when encoding the data.
+// See: https://github.com/nodejs/nan/issues/883
+const MAX_BUFFER_SIZE = 0x3fffffff - 9;
+
+function isLargeBlob(contents: Buffer | string): boolean {
+  return typeof contents === 'string'
+    ? Buffer.byteLength(contents) > MAX_BUFFER_SIZE
+    : contents.length > MAX_BUFFER_SIZE;
+}
+
 registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);

From 57343127c05c727463f7ed57fd0b9d4e65cd3784 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Wed, 3 Nov 2021 17:21:41 -0400
Subject: [PATCH 08/17] Remove stale value when auto storing a large blob

---
 packages/core/cache/src/LMDBCache.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 2f8c988fa64..059858ef5ee 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -74,8 +74,13 @@ export class LMDBCache implements Cache {
   }
 
   async setBlob(key: string, contents: Buffer | string): Promise<void> {
-    if (isLargeBlob(contents)) await this.setLargeBlob(key, contents);
-    else await this.store.put(key, contents);
+    if (isLargeBlob(contents)) {
+      // Remove the old blob if it has been 'upgraded' to large blob storage.
+      if (this.store.get(key) != null) await this.store.remove(key);
+      await this.setLargeBlob(key, contents);
+    } else {
+      await this.store.put(key, contents);
+    }
   }
 
   async getBuffer(key: string): Promise<?Buffer> {

From da0a359588334de590204b0984cf8dd4e1da9242 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Wed, 3 Nov 2021 17:33:50 -0400
Subject: [PATCH 09/17] Stream large blobs out of fs cache

---
 packages/core/cache/src/LMDBCache.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 059858ef5ee..42d27265bd5 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -58,9 +58,9 @@ export class LMDBCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    return blobToStream(
-      this.store.get(key) ?? this.fs.readFileSync(path.join(this.dir, key)),
-    );
+    let buf = this.store.get(key);
+    if (buf != null) return blobToStream(buf);
+    return this.fs.createReadStream(path.join(this.dir, key));
   }
 
   async setStream(key: string, stream: Readable): Promise<void> {

From a53b39b0123bf45f10c9e711a60a6943a1bba6cd Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 16 Nov 2021 17:05:23 -0500
Subject: [PATCH 10/17] Revert fall back to FS for large blobs in LMDBCache

---
 packages/core/cache/src/LMDBCache.js | 47 ++++++++++++----------------
 1 file changed, 20 insertions(+), 27 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 42d27265bd5..190fff3a9eb 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -44,51 +44,44 @@ export class LMDBCache implements Cache {
   }
 
   has(key: string): Promise<boolean> {
-    if (this.store.get(key) != null) return Promise.resolve(true);
-    return this.hasLargeBlob(key);
+    return Promise.resolve(this.store.get(key) != null);
   }
 
-  async get<T>(key: string): Promise<?T> {
-    let data = await this.getBuffer(key);
-    return data == null ? null : deserialize(data);
+  get<T>(key: string): Promise<?T> {
+    let data = this.store.get(key);
+    if (data == null) {
+      return Promise.resolve(null);
+    }
+
+    return Promise.resolve(deserialize(data));
   }
 
   async set(key: string, value: mixed): Promise<void> {
-    await this.setBlob(key, serialize(value));
+    await this.store.put(key, serialize(value));
   }
 
   getStream(key: string): Readable {
-    let buf = this.store.get(key);
-    if (buf != null) return blobToStream(buf);
-    return this.fs.createReadStream(path.join(this.dir, key));
+    return blobToStream(this.store.get(key));
   }
 
   async setStream(key: string, stream: Readable): Promise<void> {
-    await this.setBlob(key, await bufferStream(stream));
+    let buf = await bufferStream(stream);
+    await this.store.put(key, buf);
   }
 
-  async getBlob(key: string): Promise<Buffer> {
-    let buffer = await this.getBuffer(key);
-    if (buffer == null) throw new Error(`Key ${key} not found in cache`);
-    return buffer;
+  getBlob(key: string): Promise<Buffer> {
+    let buffer = this.store.get(key);
+    return buffer != null
+      ? Promise.resolve(buffer)
+      : Promise.reject(new Error(`Key ${key} not found in cache`));
   }
 
   async setBlob(key: string, contents: Buffer | string): Promise<void> {
-    if (isLargeBlob(contents)) {
-      // Remove the old blob if it has been 'upgraded' to large blob storage.
-      if (this.store.get(key) != null) await this.store.remove(key);
-      await this.setLargeBlob(key, contents);
-    } else {
-      await this.store.put(key, contents);
-    }
+    await this.store.put(key, contents);
   }
 
-  async getBuffer(key: string): Promise<?Buffer> {
-    let buffer = this.store.get(key);
-    if (buffer == null && (await this.hasLargeBlob(key))) {
-      buffer = await this.getLargeBlob(key);
-    }
-    return buffer;
+  getBuffer(key: string): Promise<?Buffer> {
+    return Promise.resolve(this.store.get(key));
   }
 
   hasLargeBlob(key: string): Promise<boolean> {

From ce8fe8e4202e91890db6da621fa792a3039f0fd4 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 16 Nov 2021 17:42:14 -0500
Subject: [PATCH 11/17] Prevent large blobs from being stored in cache

---
 packages/core/cache/src/LMDBCache.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 190fff3a9eb..249ffa5e2b4 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -7,6 +7,7 @@ import path from 'path';
 import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
 import {NodeFS} from '@parcel/fs';
 import {blobToStream, bufferStream} from '@parcel/utils';
+import invariant from 'assert';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
 // $FlowFixMe
@@ -57,7 +58,7 @@ export class LMDBCache implements Cache {
   }
 
   async set(key: string, value: mixed): Promise<void> {
-    await this.store.put(key, serialize(value));
+    await this.setBlob(key, serialize(value));
   }
 
   getStream(key: string): Readable {
@@ -77,6 +78,10 @@ export class LMDBCache implements Cache {
   }
 
   async setBlob(key: string, contents: Buffer | string): Promise<void> {
+    invariant(
+      !isLargeBlob(contents),
+      'Cannot store large blobs in the cache. You may want to use `setLargeBlob` instead.',
+    );
     await this.store.put(key, contents);
   }
 

From 6d9f27661eaccc2908813331032e7c7cd325c805 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 16 Nov 2021 17:52:51 -0500
Subject: [PATCH 12/17] Set/get streams to/from FS in LMDBCache

---
 packages/core/cache/src/LMDBCache.js | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 249ffa5e2b4..aad7aa16bde 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -1,12 +1,11 @@
 // @flow strict-local
-import type {Readable} from 'stream';
 import type {FilePath} from '@parcel/types';
 import type {Cache} from './types';
 
+import {Readable} from 'stream';
 import path from 'path';
 import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
 import {NodeFS} from '@parcel/fs';
-import {blobToStream, bufferStream} from '@parcel/utils';
 import invariant from 'assert';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
@@ -62,12 +61,24 @@ export class LMDBCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    return blobToStream(this.store.get(key));
+    let filename = path.join(this.dir, key);
+    if (this.fs.existsSync(filename)) {
+      return this.fs.createReadStream(filename);
+    } else {
+      // If the file doesn't exists, return an empty stream.
+      let stream = new Readable();
+      stream.push(null);
+      return stream;
+    }
   }
 
-  async setStream(key: string, stream: Readable): Promise<void> {
-    let buf = await bufferStream(stream);
-    await this.store.put(key, buf);
+  setStream(key: string, stream: Readable): Promise<void> {
+    return new Promise((resolve, reject) => {
+      stream
+        .pipe(this.fs.createWriteStream(path.join(this.dir, key)))
+        .on('error', reject)
+        .on('finish', resolve);
+    });
   }
 
   getBlob(key: string): Promise<Buffer> {

From 63f6f2cefe1027cc9b35ca9e09dabd1d6ecd1c1c Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Tue, 16 Nov 2021 19:03:06 -0500
Subject: [PATCH 13/17] Only stream large assets from cache

Before, all assets were streamed from cache regardless of size, but by
marking assets with content streams as large blobs when being written to
the cache, we can default to reading the assets into memory from cache,
and only stream the assets that were marked as large blobs.
---
 packages/core/core/src/PackagerRunner.js              |  4 ++++
 packages/core/core/src/Transformation.js              |  4 +++-
 packages/core/core/src/UncommittedAsset.js            |  1 +
 packages/core/core/src/requests/WriteBundleRequest.js | 11 +++++++++--
 packages/core/core/src/types.js                       |  1 +
 5 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/packages/core/core/src/PackagerRunner.js b/packages/core/core/src/PackagerRunner.js
index c062edc9da4..a71a7ef1e75 100644
--- a/packages/core/core/src/PackagerRunner.js
+++ b/packages/core/core/src/PackagerRunner.js
@@ -83,6 +83,7 @@ export type BundleInfo = {|
   +hashReferences: Array<string>,
   +time?: number,
   +cacheKeys: CacheKeyMap,
+  +isLargeBlob: boolean,
 |};
 
 type CacheKeyMap = {|
@@ -618,9 +619,11 @@ export default class PackagerRunner {
     let size = 0;
     let hash;
     let hashReferences = [];
+    let isLargeBlob = false;
 
     // TODO: don't replace hash references in binary files??
     if (contents instanceof Readable) {
+      isLargeBlob = true;
       let boundaryStr = '';
       let h = new Hash();
       await this.options.cache.setStream(
@@ -659,6 +662,7 @@ export default class PackagerRunner {
       hash,
       hashReferences,
       cacheKeys,
+      isLargeBlob,
     };
     await this.options.cache.set(cacheKeys.info, info);
     return info;
diff --git a/packages/core/core/src/Transformation.js b/packages/core/core/src/Transformation.js
index bdd18698f51..f2c0fab4f01 100644
--- a/packages/core/core/src/Transformation.js
+++ b/packages/core/core/src/Transformation.js
@@ -512,7 +512,9 @@ export default class Transformation {
       cachedAssets.map(async (value: AssetValue) => {
         let content =
           value.contentKey != null
-            ? this.options.cache.getStream(value.contentKey)
+            ? value.isLargeBlob
+              ? this.options.cache.getStream(value.contentKey)
+              : await this.options.cache.getBlob(value.contentKey)
             : null;
         let mapBuffer =
           value.astKey != null
diff --git a/packages/core/core/src/UncommittedAsset.js b/packages/core/core/src/UncommittedAsset.js
index ec199e6ca68..cd0fc9be2b6 100644
--- a/packages/core/core/src/UncommittedAsset.js
+++ b/packages/core/core/src/UncommittedAsset.js
@@ -135,6 +135,7 @@ export default class UncommittedAsset {
       this.value.stats.size = size;
     }
 
+    this.value.isLargeBlob = this.content instanceof Readable;
     this.value.committed = true;
   }
 
diff --git a/packages/core/core/src/requests/WriteBundleRequest.js b/packages/core/core/src/requests/WriteBundleRequest.js
index 80a997b41b7..c09fed60e93 100644
--- a/packages/core/core/src/requests/WriteBundleRequest.js
+++ b/packages/core/core/src/requests/WriteBundleRequest.js
@@ -16,7 +16,7 @@ import {HASH_REF_PREFIX, HASH_REF_REGEX} from '../constants';
 import nullthrows from 'nullthrows';
 import path from 'path';
 import {NamedBundle} from '../public/Bundle';
-import {TapStream} from '@parcel/utils';
+import {blobToStream, TapStream} from '@parcel/utils';
 import {Readable, Transform, pipeline} from 'stream';
 import {
   fromProjectPath,
@@ -123,7 +123,14 @@ async function run({input, options, api}: RunInput) {
       : {
           mode: (await inputFS.stat(mainEntry.filePath)).mode,
         };
-  let contentStream = options.cache.getStream(cacheKeys.content);
+  let contentStream: Readable;
+  if (info.isLargeBlob) {
+    contentStream = options.cache.getStream(cacheKeys.content);
+  } else {
+    contentStream = blobToStream(
+      await options.cache.getBlob(cacheKeys.content),
+    );
+  }
   let size = 0;
   contentStream = contentStream.pipe(
     new TapStream(buf => {
diff --git a/packages/core/core/src/types.js b/packages/core/core/src/types.js
index 6c498c47385..a93ffd73293 100644
--- a/packages/core/core/src/types.js
+++ b/packages/core/core/src/types.js
@@ -180,6 +180,7 @@ export type Asset = {|
   configPath?: ProjectPath,
   plugin: ?PackageName,
   configKeyPath?: string,
+  isLargeBlob?: boolean,
 |};
 
 export type InternalGlob = ProjectPath;

From 0776a9eee9ee7ee4784814d0c7f2133ad970fa84 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Thu, 18 Nov 2021 12:38:31 -0500
Subject: [PATCH 14/17] Remove default empty stream from `getStream`

---
 packages/core/cache/src/LMDBCache.js | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index aad7aa16bde..7a35c7e5880 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -61,15 +61,7 @@ export class LMDBCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    let filename = path.join(this.dir, key);
-    if (this.fs.existsSync(filename)) {
-      return this.fs.createReadStream(filename);
-    } else {
-      // If the file doesn't exists, return an empty stream.
-      let stream = new Readable();
-      stream.push(null);
-      return stream;
-    }
+    return this.fs.createReadStream(path.join(this.dir, key));
   }
 
   setStream(key: string, stream: Readable): Promise<void> {

From d9fc18fafcc2ae3b906bb3817e53d08d83df4bfa Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Thu, 18 Nov 2021 12:45:01 -0500
Subject: [PATCH 15/17] Differentiate large blobs in FSCache

This prevents `FSCache.has` and `FSCache.get` from finding large blobs,
as large blobs should be retrieved via `FSCache.getLargeBlob`
or `FSCache.getStream`.
---
 packages/core/cache/src/FSCache.js                   | 12 ++++++------
 packages/core/core/src/CommittedAsset.js             |  6 +++++-
 packages/core/core/src/PackagerRunner.js             | 12 +++++++++---
 .../core/core/src/requests/WriteBundleRequest.js     |  2 +-
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/packages/core/cache/src/FSCache.js b/packages/core/cache/src/FSCache.js
index 86132317917..43ae406ceb8 100644
--- a/packages/core/cache/src/FSCache.js
+++ b/packages/core/cache/src/FSCache.js
@@ -41,13 +41,13 @@ export class FSCache implements Cache {
   }
 
   getStream(key: string): Readable {
-    return this.fs.createReadStream(this._getCachePath(key));
+    return this.fs.createReadStream(this._getCachePath(`${key}-large`));
   }
 
   setStream(key: string, stream: Readable): Promise<void> {
     return new Promise((resolve, reject) => {
       stream
-        .pipe(this.fs.createWriteStream(this._getCachePath(key)))
+        .pipe(this.fs.createWriteStream(this._getCachePath(`${key}-large`)))
         .on('error', reject)
         .on('finish', resolve);
     });
@@ -78,15 +78,15 @@ export class FSCache implements Cache {
   }
 
   hasLargeBlob(key: string): Promise<boolean> {
-    return this.has(key);
+    return this.fs.exists(this._getCachePath(`${key}-large`));
   }
 
   getLargeBlob(key: string): Promise<Buffer> {
-    return this.getBlob(key);
+    return this.fs.readFile(this._getCachePath(`${key}-large`));
   }
 
-  setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
-    return this.setBlob(key, contents);
+  async setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
+    await this.fs.writeFile(this._getCachePath(`${key}-large`), contents);
   }
 
   async get<T>(key: string): Promise<?T> {
diff --git a/packages/core/core/src/CommittedAsset.js b/packages/core/core/src/CommittedAsset.js
index 8d97839b068..9bc94e3adf4 100644
--- a/packages/core/core/src/CommittedAsset.js
+++ b/packages/core/core/src/CommittedAsset.js
@@ -27,7 +27,11 @@ export default class CommittedAsset {
   getContent(): Blob | Promise<Buffer | string> {
     if (this.content == null) {
       if (this.value.contentKey != null) {
-        return this.options.cache.getStream(this.value.contentKey);
+        if (this.value.isLargeBlob) {
+          return this.options.cache.getStream(this.value.contentKey);
+        } else {
+          return this.options.cache.getBlob(this.value.contentKey);
+        }
       } else if (this.value.astKey != null) {
         return streamFromPromise(
           generateFromAST(this).then(({content}) => {
diff --git a/packages/core/core/src/PackagerRunner.js b/packages/core/core/src/PackagerRunner.js
index a71a7ef1e75..3e3dabcfc38 100644
--- a/packages/core/core/src/PackagerRunner.js
+++ b/packages/core/core/src/PackagerRunner.js
@@ -597,7 +597,9 @@ export default class PackagerRunner {
     let contentKey = PackagerRunner.getContentKey(cacheKey);
     let mapKey = PackagerRunner.getMapKey(cacheKey);
 
-    let contentExists = await this.options.cache.has(contentKey);
+    let isLargeBlob = await this.options.cache.hasLargeBlob(contentKey);
+    let contentExists =
+      isLargeBlob || (await this.options.cache.has(contentKey));
     if (!contentExists) {
       return null;
     }
@@ -605,8 +607,12 @@ export default class PackagerRunner {
     let mapExists = await this.options.cache.has(mapKey);
 
     return {
-      contents: this.options.cache.getStream(contentKey),
-      map: mapExists ? this.options.cache.getStream(mapKey) : null,
+      contents: isLargeBlob
+        ? this.options.cache.getStream(contentKey)
+        : blobToStream(this.options.cache.getBlob(contentKey)),
+      map: mapExists
+        ? blobToStream(await this.options.cache.getBlob(mapKey))
+        : null,
     };
   }
 
diff --git a/packages/core/core/src/requests/WriteBundleRequest.js b/packages/core/core/src/requests/WriteBundleRequest.js
index c09fed60e93..b87e7a8afd3 100644
--- a/packages/core/core/src/requests/WriteBundleRequest.js
+++ b/packages/core/core/src/requests/WriteBundleRequest.js
@@ -166,7 +166,7 @@ async function run({input, options, api}: RunInput) {
     (await options.cache.has(mapKey))
   ) {
     await writeFiles(
-      options.cache.getStream(mapKey),
+      blobToStream(await options.cache.getBlob(mapKey)),
       info,
       hashRefToNameHash,
       options,

From ed1b6020f0cd94a20154602beb99eea9745f570c Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Thu, 18 Nov 2021 17:06:57 -0500
Subject: [PATCH 16/17] Fix type error

---
 packages/core/core/src/PackagerRunner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/core/core/src/PackagerRunner.js b/packages/core/core/src/PackagerRunner.js
index 3e3dabcfc38..a2b778e05a2 100644
--- a/packages/core/core/src/PackagerRunner.js
+++ b/packages/core/core/src/PackagerRunner.js
@@ -609,7 +609,7 @@ export default class PackagerRunner {
     return {
       contents: isLargeBlob
         ? this.options.cache.getStream(contentKey)
-        : blobToStream(this.options.cache.getBlob(contentKey)),
+        : blobToStream(await this.options.cache.getBlob(contentKey)),
       map: mapExists
         ? blobToStream(await this.options.cache.getBlob(mapKey))
         : null,

From 3f3c8314f8f1f87b2f0924990c7ea0fcc71559d1 Mon Sep 17 00:00:00 2001
From: Eric Eldredge <lettertwo@gmail.com>
Date: Mon, 22 Nov 2021 13:15:47 -0500
Subject: [PATCH 17/17] Remove vestigial isLargeBlob check

---
 packages/core/cache/src/LMDBCache.js | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/packages/core/cache/src/LMDBCache.js b/packages/core/cache/src/LMDBCache.js
index 7a35c7e5880..16ab8d54f91 100644
--- a/packages/core/cache/src/LMDBCache.js
+++ b/packages/core/cache/src/LMDBCache.js
@@ -6,7 +6,6 @@ import {Readable} from 'stream';
 import path from 'path';
 import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
 import {NodeFS} from '@parcel/fs';
-import invariant from 'assert';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
 // $FlowFixMe
@@ -81,10 +80,6 @@ export class LMDBCache implements Cache {
   }
 
   async setBlob(key: string, contents: Buffer | string): Promise<void> {
-    invariant(
-      !isLargeBlob(contents),
-      'Cannot store large blobs in the cache. You may want to use `setLargeBlob` instead.',
-    );
     await this.store.put(key, contents);
   }
 
@@ -105,17 +100,4 @@ export class LMDBCache implements Cache {
   }
 }
 
-// lmbd-store decodes cached binary data into a Node Buffer
-// via `Nan::NewBuffer`, which enforces a max size of ~1GB.
-// We subtract 9 bytes to account for any compression heaader
-// added by lmbd-store when encoding the data.
-// See: https://github.com/nodejs/nan/issues/883
-const MAX_BUFFER_SIZE = 0x3fffffff - 9;
-
-function isLargeBlob(contents: Buffer | string): boolean {
-  return typeof contents === 'string'
-    ? Buffer.byteLength(contents) > MAX_BUFFER_SIZE
-    : contents.length > MAX_BUFFER_SIZE;
-}
-
 registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);