diff --git a/Cargo.lock b/Cargo.lock index bd89f185b39..7d47aa81c96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -225,6 +225,15 @@ dependencies = [ "vec_map", ] +[[package]] +name = "cloudabi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" +dependencies = [ + "bitflags", +] + [[package]] name = "cloudflare-zlib" version = "0.2.9" @@ -437,7 +446,7 @@ checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", + "redox_syscall 0.2.10", "winapi", ] @@ -1049,6 +1058,7 @@ version = "0.1.0" dependencies = [ "js-sys", "parcel-js-swc-core", + "parking_lot_core", "serde", "serde-wasm-bindgen", "wasm-bindgen", @@ -1056,9 +1066,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.11.2" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", "lock_api", @@ -1067,14 +1077,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" dependencies = [ - "cfg-if 1.0.0", + "cfg-if 0.1.10", + "cloudabi", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", "winapi", ] @@ -1288,6 +1299,12 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + [[package]] name = "redox_syscall" version = "0.2.10" diff --git a/packages/core/cache/package.json b/packages/core/cache/package.json index 9f88d43d7de..5d7a5ddd1a0 100644 --- a/packages/core/cache/package.json +++ b/packages/core/cache/package.json @@ -30,5 +30,12 @@ }, "peerDependencies": { "@parcel/core": "^2.0.0" + }, + "devDependencies": { + "idb": "^5.0.8" + }, + "browser": { + "./src/IDBCache.js": "./src/IDBCache.browser.js", + "./src/LMDBCache.js": false } } diff --git a/packages/core/cache/src/IDBCache.browser.js b/packages/core/cache/src/IDBCache.browser.js new file mode 100644 index 00000000000..382d0b9c463 --- /dev/null +++ b/packages/core/cache/src/IDBCache.browser.js @@ -0,0 +1,120 @@ +// @flow strict-local +import type {Cache} from './types'; + +import {Readable} from 'stream'; +import {serialize, deserialize, registerSerializableClass} from '@parcel/core'; +import {bufferStream} from '@parcel/utils'; +// $FlowFixMe[untyped-import] +import packageJson from '../package.json'; +// $FlowFixMe[untyped-import] +import {openDB} from 'idb'; + +const STORE_NAME = 'cache'; + +export class IDBCache implements Cache { + // $FlowFixMe + store: any; + + constructor() { + this.store = openDB('REPL-parcel-cache', 1, { + upgrade(db) { + db.createObjectStore(STORE_NAME); + }, + blocked() {}, + blocking() {}, + terminated() {}, + }); + } + + ensure(): Promise { + return Promise.resolve(); + } + + serialize(): {||} { + return {...null}; + } + + static deserialize(): IDBCache { + return new IDBCache(); + } + + has(key: string): Promise { + return Promise.resolve(this.store.get(key) != null); + } + + async get(key: string): Promise { + let data = await (await this.store).get(STORE_NAME, key); + if (data == null) { + return null; + } + + return Promise.resolve(deserialize(data)); + } + + async set(key: string, value: mixed): Promise { + await (await this.store).put(STORE_NAME, serialize(value), key); + } + + getStream(key: string): Readable { + let dataPromise = this.store + .then(s => s.get(STORE_NAME, key)) + .then(d => Buffer.from(d)) + .catch(e => e); + const stream = new Readable({ + // $FlowFixMe(incompatible-call) + async read() { + let data = await dataPromise; + if (data instanceof Error) { + stream.emit('error', data); + } else { + stream.push(Buffer.from(data)); + stream.push(null); + } + }, + }); + + return stream; + } + + async setStream(key: string, stream: Readable): Promise { + let buf = await bufferStream(stream); + await (await this.store).put(STORE_NAME, buf, key); + } + + async getBlob(key: string): Promise { + let data = await (await this.store).get(STORE_NAME, key); + if (data == null) { + return Promise.reject(new Error(`Key ${key} not found in cache`)); + } + return Buffer.from(data.buffer); + } + + async setBlob(key: string, contents: Buffer | string): Promise { + let data = + contents instanceof Uint8Array ? contents : Buffer.from(contents); + await (await this.store).put(STORE_NAME, data, key); + } + + async getBuffer(key: string): Promise { + let data = await (await this.store).get(STORE_NAME, key); + if (data == null) { + return null; + } + + return Buffer.from(data.buffer); + } + + hasLargeBlob(key: string): Promise { + return this.has(key); + } + + getLargeBlob(key: string): Promise { + return this.getBlob(key); + } + + setLargeBlob(key: string, contents: Buffer | string): Promise { + return this.setBlob(key, contents); + } +} + +registerSerializableClass(`${packageJson.version}:IDBCache`, IDBCache); diff --git a/packages/core/cache/src/IDBCache.js b/packages/core/cache/src/IDBCache.js new file mode 100644 index 00000000000..515e8480a6b --- /dev/null +++ b/packages/core/cache/src/IDBCache.js @@ -0,0 +1,9 @@ +// @flow strict-local +import type {Cache} from './types'; + +// $FlowFixMe +export class IDBCache implements Cache { + constructor() { + throw new Error('IDBCache is only supported in the browser'); + } +} diff --git a/packages/core/cache/src/index.js b/packages/core/cache/src/index.js index a38d6ae1da6..a83c15f59cc 100644 --- a/packages/core/cache/src/index.js +++ b/packages/core/cache/src/index.js @@ -2,3 +2,4 @@ export type {Cache} from './types'; export * from './LMDBCache'; export * from './FSCache'; +export * from './IDBCache'; diff --git a/packages/core/core/package.json b/packages/core/core/package.json index e198bc99124..8296f1195aa 100644 --- a/packages/core/core/package.json +++ b/packages/core/core/package.json @@ -45,6 +45,7 @@ "dotenv-expand": "^5.1.0", "json-source-map": "^0.6.1", "json5": "^1.0.1", + "msgpackr": "^1.5.1", "micromatch": "^4.0.2", "nullthrows": "^1.1.1", "semver": "^5.7.1" @@ -52,5 +53,8 @@ "devDependencies": { "graphviz": "^0.0.9", "tempy": "^0.2.1" + }, + "browser": { + "./src/serializerCore.js": "./src/serializerCore.browser.js" } } diff --git a/packages/core/core/src/PackagerRunner.js b/packages/core/core/src/PackagerRunner.js index a2b778e05a2..3c18a16c10b 100644 --- a/packages/core/core/src/PackagerRunner.js +++ b/packages/core/core/src/PackagerRunner.js @@ -648,10 +648,11 @@ export default class PackagerRunner { ); hash = h.finish(); } else if (typeof contents === 'string') { - size = Buffer.byteLength(contents); - hash = hashString(contents); + let buffer = Buffer.from(contents); + size = buffer.byteLength; + hash = hashBuffer(buffer); hashReferences = contents.match(HASH_REF_REGEX) ?? []; - await this.options.cache.setBlob(cacheKeys.content, contents); + await this.options.cache.setBlob(cacheKeys.content, buffer); } else { size = contents.length; hash = hashBuffer(contents); diff --git a/packages/core/core/src/serializer.js b/packages/core/core/src/serializer.js index 57b770c533d..b93fc71b141 100644 --- a/packages/core/core/src/serializer.js +++ b/packages/core/core/src/serializer.js @@ -1,11 +1,8 @@ // @flow -import v8 from 'v8'; import {createBuildCache} from './buildCache'; +import {serializeRaw, deserializeRaw} from './serializerCore'; -// $FlowFixMe - Flow doesn't know about this method yet -export let serializeRaw = v8.serialize; -// $FlowFixMe - Flow doesn't know about this method yet -export let deserializeRaw = v8.deserialize; +export {serializeRaw, deserializeRaw} from './serializerCore'; const nameToCtor: Map> = new Map(); const ctorToName: Map, string> = new Map(); @@ -55,7 +52,8 @@ function shallowCopy(object: any) { function isBuffer(object) { return ( object.buffer instanceof ArrayBuffer || - object.buffer instanceof SharedArrayBuffer + (typeof SharedArrayBuffer !== 'undefined' && + object.buffer instanceof SharedArrayBuffer) ); } diff --git a/packages/core/core/src/serializerCore.browser.js b/packages/core/core/src/serializerCore.browser.js new file mode 100644 index 00000000000..e587330814f --- /dev/null +++ b/packages/core/core/src/serializerCore.browser.js @@ -0,0 +1,8 @@ +// @flow +import {Buffer} from 'buffer'; +import * as msgpackr from 'msgpackr'; + +let encoder = new msgpackr.Encoder({structuredClone: true}); + +export let serializeRaw: any => Buffer = v => Buffer.from(encoder.encode(v)); +export let deserializeRaw: Buffer => any = v => encoder.decode(v); diff --git a/packages/core/core/src/serializerCore.js b/packages/core/core/src/serializerCore.js new file mode 100644 index 00000000000..124a042861f --- /dev/null +++ b/packages/core/core/src/serializerCore.js @@ -0,0 +1,7 @@ +// @flow +import v8 from 'v8'; + +// $FlowFixMe - Flow doesn't know about this method yet +export let serializeRaw: any => Buffer = v8.serialize; +// $FlowFixMe - Flow doesn't know about this method yet +export let deserializeRaw: Buffer => any = v8.deserialize; diff --git a/packages/core/fs/package.json b/packages/core/fs/package.json index 5e0f2ceb9d6..ad7ca36b6cc 100644 --- a/packages/core/fs/package.json +++ b/packages/core/fs/package.json @@ -40,5 +40,8 @@ }, "peerDependencies": { "@parcel/core": "^2.0.0" + }, + "browser": { + "./src/NodeFS.js": "./src/NodeFS.browser.js" } } diff --git a/packages/core/fs/src/MemoryFS.js b/packages/core/fs/src/MemoryFS.js index 93e25fe701b..17afdcc4f75 100644 --- a/packages/core/fs/src/MemoryFS.js +++ b/packages/core/fs/src/MemoryFS.js @@ -11,6 +11,7 @@ import type { import path from 'path'; import {Readable, Writable} from 'stream'; import {registerSerializableClass} from '@parcel/core'; +import {SharedBuffer} from '@parcel/utils'; import packageJSON from '../package.json'; import WorkerFarm, {Handle} from '@parcel/workers'; import nullthrows from 'nullthrows'; @@ -904,15 +905,12 @@ class Directory extends Entry { } function makeShared(contents: Buffer | string): Buffer { - if ( - typeof contents !== 'string' && - contents.buffer instanceof SharedArrayBuffer - ) { + if (typeof contents !== 'string' && contents.buffer instanceof SharedBuffer) { return contents; } let length = Buffer.byteLength(contents); - let shared = new SharedArrayBuffer(length); + let shared = new SharedBuffer(length); let buffer = Buffer.from(shared); if (typeof contents === 'string') { buffer.write(contents); diff --git a/packages/core/fs/src/NodeFS.browser.js b/packages/core/fs/src/NodeFS.browser.js new file mode 100644 index 00000000000..43454dc61f8 --- /dev/null +++ b/packages/core/fs/src/NodeFS.browser.js @@ -0,0 +1,9 @@ +// @flow +import type {FileSystem} from './types'; + +// $FlowFixMe[prop-missing] handled by the throwing constructor +export class NodeFS implements FileSystem { + constructor() { + throw new Error("NodeFS isn't available in the browser"); + } +} diff --git a/packages/core/graph/src/AdjacencyList.js b/packages/core/graph/src/AdjacencyList.js index 6f8d4c28f4b..5db0b5a320a 100644 --- a/packages/core/graph/src/AdjacencyList.js +++ b/packages/core/graph/src/AdjacencyList.js @@ -1,6 +1,7 @@ // @flow import assert from 'assert'; import nullthrows from 'nullthrows'; +import {SharedBuffer} from '@parcel/utils'; import {fromNodeId, toNodeId} from './types'; import {ALL_EDGE_TYPES, type NullEdgeType, type AllEdgeTypes} from './Graph'; import type {NodeId} from './types'; @@ -610,9 +611,7 @@ export class SharedTypeMap let CAPACITY = SharedTypeMap.#CAPACITY; // $FlowFixMe[incompatible-call] this.data = new Uint32Array( - new SharedArrayBuffer( - this.getLength(capacityOrData) * BYTES_PER_ELEMENT, - ), + new SharedBuffer(this.getLength(capacityOrData) * BYTES_PER_ELEMENT), ); this.data[CAPACITY] = capacityOrData; } else { diff --git a/packages/core/package-manager/package.json b/packages/core/package-manager/package.json index d03650aacec..5ab91f4291b 100644 --- a/packages/core/package-manager/package.json +++ b/packages/core/package-manager/package.json @@ -39,5 +39,10 @@ }, "peerDependencies": { "@parcel/core": "^2.0.0" + }, + "browser": { + "./src/Npm.js": false, + "./src/Pnpm.js": false, + "./src/Yarn.js": false } } diff --git a/packages/core/utils/package.json b/packages/core/utils/package.json index 73335e473cf..1e752b9a81e 100644 --- a/packages/core/utils/package.json +++ b/packages/core/utils/package.json @@ -45,5 +45,10 @@ "devDependencies": { "@babel/plugin-transform-flow-strip-types": "^7.2.0", "random-int": "^1.0.0" + }, + "browser": { + "./src/generateCertificate.js": false, + "./src/http-server.js": false, + "./src/openInBrowser.js": false } } diff --git a/packages/core/utils/src/.babelrc b/packages/core/utils/src/.babelrc deleted file mode 100644 index be26495ef30..00000000000 --- a/packages/core/utils/src/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@parcel/babel-preset"] -} diff --git a/packages/core/utils/src/blob.js b/packages/core/utils/src/blob.js index c84acd2a2c8..50988d010d4 100644 --- a/packages/core/utils/src/blob.js +++ b/packages/core/utils/src/blob.js @@ -2,6 +2,7 @@ import type {Blob} from '@parcel/types'; +import {Buffer} from 'buffer'; import {bufferStream} from './'; import {Readable} from 'stream'; diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index be4d656b0df..4b6b20f24d8 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -41,6 +41,7 @@ export {DefaultMap, DefaultWeakMap} from './DefaultMap'; export {makeDeferredWithPromise} from './Deferred'; export {isGlob, isGlobMatch, globSync, glob} from './glob'; export {hashStream, hashObject, hashFile} from './hash'; +export {SharedBuffer} from './shared-buffer'; export {fuzzySearch} from './schema'; export {createHTTPServer} from './http-server'; export {normalizePath, normalizeSeparators, relativePath} from './path'; diff --git a/packages/core/utils/src/shared-buffer.js b/packages/core/utils/src/shared-buffer.js new file mode 100644 index 00000000000..ba7631fbf78 --- /dev/null +++ b/packages/core/utils/src/shared-buffer.js @@ -0,0 +1,24 @@ +// @flow +/* global MessageChannel:readonly */ + +export let SharedBuffer: Class | Class; + +// $FlowFixMe[prop-missing] +if (process.browser) { + SharedBuffer = ArrayBuffer; + // Safari has removed the constructor + if (typeof SharedArrayBuffer !== 'undefined') { + let channel = new MessageChannel(); + try { + // Firefox might throw when sending the Buffer over a MessagePort + channel.port1.postMessage(new SharedArrayBuffer(0)); + SharedBuffer = SharedArrayBuffer; + } catch (_) { + // NOOP + } + channel.port1.close(); + channel.port2.close(); + } +} else { + SharedBuffer = SharedArrayBuffer; +} diff --git a/packages/core/workers/package.json b/packages/core/workers/package.json index 6248ec76342..8284be26cde 100644 --- a/packages/core/workers/package.json +++ b/packages/core/workers/package.json @@ -30,5 +30,10 @@ }, "peerDependencies": { "@parcel/core": "^2.0.0" + }, + "browser": { + "./src/cpuCount.js": false, + "./src/process/ProcessWorker.js": false, + "./src/threads/ThreadsWorker.js": false } } diff --git a/packages/reporters/dev-server/src/.babelrc b/packages/reporters/dev-server/src/.babelrc deleted file mode 100644 index be26495ef30..00000000000 --- a/packages/reporters/dev-server/src/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@parcel/babel-preset"] -} diff --git a/packages/transformers/js/wasm/Cargo.toml b/packages/transformers/js/wasm/Cargo.toml index 822b1b44b42..837c416c91e 100644 --- a/packages/transformers/js/wasm/Cargo.toml +++ b/packages/transformers/js/wasm/Cargo.toml @@ -13,3 +13,7 @@ parcel-js-swc-core = { path = "../core" } serde = "1" serde-wasm-bindgen = "0.3.0" wasm-bindgen = "0.2" + +# https://github.com/Amanieu/parking_lot/issues/269 +[target.'cfg(target_arch = "wasm32")'.dependencies] +parking_lot_core = "=0.8.0" diff --git a/packages/transformers/typescript-tsc/.babelrc b/packages/transformers/typescript-tsc/.babelrc deleted file mode 100644 index be26495ef30..00000000000 --- a/packages/transformers/typescript-tsc/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@parcel/babel-preset"] -} diff --git a/packages/utils/hash/browser.js b/packages/utils/hash/browser.js index 5012a2da1a4..7f75a142370 100644 --- a/packages/utils/hash/browser.js +++ b/packages/utils/hash/browser.js @@ -1,5 +1,5 @@ // @flow -const xxhash = require('xxhash-wasm'); +import xxhash from 'xxhash-wasm'; let h64, h64Raw; module.exports.init = (xxhash().then(xxh => { diff --git a/packages/utils/hash/package.json b/packages/utils/hash/package.json index 0180541720d..0f36288823d 100644 --- a/packages/utils/hash/package.json +++ b/packages/utils/hash/package.json @@ -33,7 +33,7 @@ }, "dependencies": { "detect-libc": "^1.0.3", - "xxhash-wasm": "^0.4.1" + "xxhash-wasm": "^0.4.2" }, "devDependencies": { "@napi-rs/cli": "1.0.4", diff --git a/packages/validators/eslint/.babelrc b/packages/validators/eslint/.babelrc deleted file mode 100644 index be26495ef30..00000000000 --- a/packages/validators/eslint/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@parcel/babel-preset"] -} diff --git a/packages/validators/typescript/.babelrc b/packages/validators/typescript/.babelrc deleted file mode 100644 index be26495ef30..00000000000 --- a/packages/validators/typescript/.babelrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "presets": ["@parcel/babel-preset"] -} diff --git a/yarn.lock b/yarn.lock index a3c044867d7..96e8cc101ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7189,6 +7189,11 @@ icss-utils@^4.0.0, icss-utils@^4.1.1: dependencies: postcss "^7.0.14" +idb@^5.0.8: + version "5.0.8" + resolved "https://registry.yarnpkg.com/idb/-/idb-5.0.8.tgz#5f2b72a69267960d222a5f104053625f203fdbb2" + integrity sha512-K9xInRkVbT3ZsYimD2KVj6B4E93IBvOjEQTryu99WuuN7G+7x3SzA79+yubbX0QRN9V64Gi+L+ulG5QYTVydOg== + ieee754@^1.1.4: version "1.1.13" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" @@ -9045,7 +9050,7 @@ msgpackr-extract@^1.0.14: nan "^2.14.2" node-gyp-build "^4.2.3" -msgpackr@^1.5.0: +msgpackr@^1.5.0, msgpackr@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/msgpackr/-/msgpackr-1.5.1.tgz#2a8e39d25458406034b8cb50dc7d6a7abd3dfff2" integrity sha512-I1CXFG8BYYSeIhtDlHpUVMsdDiyvP9JAh1d9QoBnkPx3ETPeH/1lR14hweM9GETs09wCWlaOyhtXxIc9boxAAA== @@ -14066,10 +14071,10 @@ xtend@^2.1.2: resolved "https://registry.yarnpkg.com/xtend/-/xtend-2.2.0.tgz#eef6b1f198c1c8deafad8b1765a04dad4a01c5a9" integrity sha1-7vax8ZjByN6vrYsXZaBNrUoBxak= -xxhash-wasm@^0.4.1: - version "0.4.1" - resolved "https://registry.yarnpkg.com/xxhash-wasm/-/xxhash-wasm-0.4.1.tgz#4795fdf243d01f03a17ac37d2ed92d001f3a1b5e" - integrity sha512-sAaACjH5Th5O2Y1Pl6Mm03bHdie8htTm7ZG146by2ITXuxD1Ksx46ZEOYaDhtlCY3fHrmDfdvzTOGzO1R00COA== +xxhash-wasm@^0.4.2: + version "0.4.2" + resolved "https://registry.yarnpkg.com/xxhash-wasm/-/xxhash-wasm-0.4.2.tgz#752398c131a4dd407b5132ba62ad372029be6f79" + integrity sha512-/eyHVRJQCirEkSZ1agRSCwriMhwlyUcFkXD5TPVSLP+IPzjsqMVzZwdoczLp1SoQU0R3dxz1RpIK+4YNQbCVOA== y18n@^3.2.1: version "3.2.2"