Add large blob storage to Cache

packages/core/cache/src/FSCache.js

@@ -77,6 +77,18 @@ export class FSCache implements Cache {
     }
   }
 
+  hasLargeBlob(key: string): Promise<boolean> {
+    return this.has(key);
+  }
+
+  getLargeBlob(key: string): Promise<Buffer> {
+    return this.getBlob(key);
+  }
+
+  setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
+    return this.setBlob(key, contents);
+  }
+
   async get<T>(key: string): Promise<?T> {
     try {
       let data = await this.fs.readFile(this._getCachePath(key));

packages/core/cache/src/LMDBCache.js

@@ -1,21 +1,30 @@
 // @flow strict-local
 import type {Readable} from 'stream';
 import type {FilePath} from '@parcel/types';
+import type {FileSystem} from '@parcel/fs';
 import type {Cache} from './types';
 
-import {serialize, deserialize, registerSerializableClass} from '@parcel/core';
+import path from 'path';
+import {
+  serialize,
+  deserialize,
+  prepareForSerialization,
+  registerSerializableClass,
+} from '@parcel/core';
 import {blobToStream, bufferStream} from '@parcel/utils';
 // flowlint-next-line untyped-import:off
 import packageJson from '../package.json';
 // $FlowFixMe
 import lmdb from 'lmdb-store';
 
 export class LMDBCache implements Cache {
+  fs: FileSystem;
   dir: FilePath;
   // $FlowFixMe
   store: any;
 
-  constructor(cacheDir: FilePath) {
+  constructor(fs: FileSystem, cacheDir: FilePath) {
+    this.fs = fs;
     this.dir = cacheDir;
 
     this.store = lmdb.open(cacheDir, {
@@ -29,14 +38,15 @@ export class LMDBCache implements Cache {
     return Promise.resolve();
   }
 
-  serialize(): {|dir: FilePath|} {
+  serialize(): {|fs: FileSystem, dir: FilePath|} {
     return {
+      fs: prepareForSerialization(this.fs),
       dir: this.dir,
     };
   }
 
-  static deserialize(opts: {|dir: FilePath|}): LMDBCache {
-    return new LMDBCache(opts.dir);
+  static deserialize(opts: {|fs: FileSystem, dir: FilePath|}): LMDBCache {
+    return new LMDBCache(opts.fs, opts.dir);
   }
 
   has(key: string): Promise<boolean> {
@@ -79,6 +89,18 @@ export class LMDBCache implements Cache {
   getBuffer(key: string): Promise<?Buffer> {
     return Promise.resolve(this.store.get(key));
   }
+
+  hasLargeBlob(key: string): Promise<boolean> {
+    return this.fs.exists(path.join(this.dir, key));
+  }
+
+  getLargeBlob(key: string): Promise<Buffer> {
+    return this.fs.readFile(path.join(this.dir, key));
+  }
+
+  async setLargeBlob(key: string, contents: Buffer | string): Promise<void> {
+    await this.fs.writeFile(path.join(this.dir, key), contents);
+  }
 }
 
 registerSerializableClass(`${packageJson.version}:LMDBCache`, LMDBCache);

packages/core/cache/src/types.js

@@ -10,5 +10,8 @@ export interface Cache {
   setStream(key: string, stream: Readable): Promise<void>;
   getBlob(key: string): Promise<Buffer>;
   setBlob(key: string, contents: Buffer | string): Promise<void>;
+  hasLargeBlob(key: string): Promise<boolean>;
+  getLargeBlob(key: string): Promise<Buffer>;
+  setLargeBlob(key: string, contents: Buffer | string): Promise<void>;
   getBuffer(key: string): Promise<?Buffer>;
 }

packages/core/core/src/RequestTracker.js

@@ -24,6 +24,7 @@ import {
 } from '@parcel/utils';
 import {hashString} from '@parcel/hash';
 import {ContentGraph} from '@parcel/graph';
+import {deserialize, serialize} from './serializer';
 import {assertSignalNotAborted, hashFromOption} from './utils';
 import {
   type ProjectPath,
@@ -1039,7 +1040,9 @@ export default class RequestTracker {
       }
     }
 
-    promises.push(this.options.cache.set(requestGraphKey, this.graph));
+    promises.push(
+      this.options.cache.setLargeBlob(requestGraphKey, serialize(this.graph)),
+    );
 
     let opts = getWatcherOptions(this.options);
     let snapshotPath = path.join(this.options.cacheDir, snapshotKey + '.txt');
@@ -1083,9 +1086,10 @@ async function loadRequestGraph(options): Async<RequestGraph> {
 
   let cacheKey = getCacheKey(options);
   let requestGraphKey = hashString(`${cacheKey}:requestGraph`);
-  let requestGraph = await options.cache.get<RequestGraph>(requestGraphKey);
-
-  if (requestGraph) {
+  if (await options.cache.hasLargeBlob(requestGraphKey)) {
+    let requestGraph: RequestGraph = deserialize(
+      await options.cache.getLargeBlob(requestGraphKey),
+    );
     let opts = getWatcherOptions(options);
     let snapshotKey = hashString(`${cacheKey}:snapshot`);
     let snapshotPath = path.join(options.cacheDir, snapshotKey + '.txt');

packages/core/core/src/resolveOptions.js

@@ -74,7 +74,7 @@ export default async function resolveOptions(
   let cache =
     initialOptions.cache ??
     (outputFS instanceof NodeFS
-      ? new LMDBCache(cacheDir)
+      ? new LMDBCache(outputFS, cacheDir)
       : new FSCache(outputFS, cacheDir));
 
   let mode = initialOptions.mode ?? 'development';