diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afdd321d79..20d4fb7910 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,61 @@ jobs: - name: Run yarn test run: yarn test + test-async-hooks: + name: test-async-hooks + + runs-on: ${{ matrix.platform }} + strategy: + fail-fast: false + matrix: + node-version: + - '14.2' # last version before promise fast-path without destroy + - '14.17' # last version before unconditional promise fast-path + - '14.18' # first version after unconditional promise fast-path + - '16.1' # last version before some significant promise hooks changes + - '16.5' # last version before unconditional promise fast-path + - '16.6' # first version after unconditional promise fast-path + platform: + - ubuntu-latest + +# begin macro + + steps: + + - name: Checkout + uses: actions/checkout@v2 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + + - name: Echo node version + run: node --version + + - name: Get yarn cache directory path + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: Cache npm modules + uses: actions/cache@v1 + id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`) + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + + - name: Install dependencies + run: yarn --frozen-lockfile + +# end macro + + - name: Run yarn build + run: yarn build + + - name: Run yarn test (@endo/init) + working-directory: packages/init + run: yarn test + cover: name: cover diff --git a/packages/init/node-async_hooks-patch.js b/packages/init/node-async_hooks-patch.js new file mode 100644 index 0000000000..6fe0ce4414 --- /dev/null +++ b/packages/init/node-async_hooks-patch.js @@ -0,0 +1,3 @@ +import { setup } from './node-async_hooks.js'; + +setup(true); diff --git a/packages/init/node-async_hooks-symbols.d.ts b/packages/init/node-async_hooks-symbols.d.ts new file mode 100644 index 0000000000..09fd7027de --- /dev/null +++ b/packages/init/node-async_hooks-symbols.d.ts @@ -0,0 +1,11 @@ +// @ts-check + +export {}; + +declare global { + interface SymbolConstructor { + readonly nodeAsyncHooksAsyncId: unique symbol; + readonly nodeAsyncHooksTriggerAsyncId: unique symbol; + readonly nodeAsyncHooksDestroyed: unique symbol; + } +} diff --git a/packages/init/node-async_hooks.js b/packages/init/node-async_hooks.js new file mode 100644 index 0000000000..b102c19000 --- /dev/null +++ b/packages/init/node-async_hooks.js @@ -0,0 +1,226 @@ +// @ts-check + +import { createHook, AsyncResource } from 'async_hooks'; + +/// + +const asyncHooksWellKnownNameFromDescription = { + async_id_symbol: 'nodeAsyncHooksAsyncId', + trigger_async_id_symbol: 'nodeAsyncHooksTriggerAsyncId', + destroyed: 'nodeAsyncHooksDestroyed', +}; + +const promiseAsyncHookFallbackStates = new WeakMap(); + +const setAsyncSymbol = (description, symbol) => { + const wellKnownName = asyncHooksWellKnownNameFromDescription[description]; + if (!wellKnownName) { + throw new Error('Unknown symbol'); + } else if (!Symbol[wellKnownName]) { + Symbol[wellKnownName] = symbol; + return true; + } else if (Symbol[wellKnownName] !== symbol) { + // console.warn( + // `Found duplicate ${description}:`, + // symbol, + // Symbol[wellKnownName], + // ); + return false; + } else { + return true; + } +}; + +// We can get the `async_id_symbol` and `trigger_async_id_symbol` through a +// simple instantiation of async_hook.AsyncResource, which causes little side +// effects. These are the 2 symbols that may be late bound, aka after the promise +// is returned to the program and would normally be frozen. +const findAsyncSymbolsFromAsyncResource = () => { + let found = 0; + Object.getOwnPropertySymbols(new AsyncResource('Bootstrap')).forEach(sym => { + const { description } = sym; + if (description in asyncHooksWellKnownNameFromDescription) { + if (setAsyncSymbol(description, sym)) { + found += 1; + } + } + }); + return found; +}; + +// To get the `destroyed` symbol installed on promises by async_hooks, +// the only option is to create and enable an AsyncHook. +// Different versions of Node handle this in various ways. +const findAsyncSymbolsFromPromiseCreateHook = () => { + const bootstrapData = []; + + { + const bootstrapHook = createHook({ + init(asyncId, type, triggerAsyncId, resource) { + if (type !== 'PROMISE') return; + // console.log('Bootstrap', asyncId, triggerAsyncId, resource); + bootstrapData.push({ asyncId, triggerAsyncId, resource }); + }, + destroy(_asyncId) { + // Needs to be present to trigger the addition of the destroyed symbol + }, + }); + + bootstrapHook.enable(); + // Use a never resolving promise to avoid triggering settlement hooks + const trigger = new Promise(() => {}); + bootstrapHook.disable(); + + // In some versions of Node, async_hooks don't give access to the resource + // itself, but to a "wrapper" which is basically hooks metadata for the promise + const promisesData = bootstrapData.filter( + ({ resource }) => Promise.resolve(resource) === resource, + ); + bootstrapData.length = 0; + const { length } = promisesData; + if (length > 1) { + // console.warn('Found multiple potential candidates'); + } + + const promiseData = promisesData.find( + ({ resource }) => resource === trigger, + ); + if (promiseData) { + bootstrapData.push(promiseData); + } else if (length) { + // console.warn('No candidates matched'); + } + } + + if (bootstrapData.length) { + // Normally all promise hooks are disabled in a subsequent microtask + // That means Node versions that modify promises at init will still + // trigger our proto hooks for promises created in this turn + // The following trick will disable the internal promise init hook + // However, only do this for destroy modifying versions, since some versions + // only modify promises if no destroy hook is requested, and do not correctly + // reset the internal init promise hook in those case. (e.g. v14.16.2) + const resetHook = createHook({}); + resetHook.enable(); + resetHook.disable(); + + const { asyncId, triggerAsyncId, resource } = bootstrapData.pop(); + const symbols = Object.getOwnPropertySymbols(resource); + // const { length } = symbols; + let found = 0; + // if (length !== 3) { + // console.error(`Found ${length} symbols on promise:`, ...symbols); + // } + symbols.forEach(symbol => { + const value = resource[symbol]; + let type; + if (value === asyncId) { + type = 'async_id_symbol'; + } else if (value === triggerAsyncId) { + type = 'trigger_async_id_symbol'; + } else if (typeof value === 'object' && 'destroyed' in value) { + type = 'destroyed'; + } else { + // console.error(`Unexpected symbol`, symbol); + return; + } + + if (setAsyncSymbol(type, symbol)) { + found += 1; + } + }); + return found; + } else { + // This node version is not mutating promises + return -2; + } +}; + +const getAsyncHookFallbackState = (promise, create) => { + let state = promiseAsyncHookFallbackStates.get(promise); + if (!state && create) { + state = { + [Symbol.nodeAsyncHooksAsyncId]: undefined, + [Symbol.nodeAsyncHooksTriggerAsyncId]: undefined, + }; + if (Symbol.nodeAsyncHooksDestroyed) { + state[Symbol.nodeAsyncHooksDestroyed] = undefined; + } + promiseAsyncHookFallbackStates.set(promise, state); + } + return state; +}; + +const setAsyncIdFallback = (promise, symbol, value) => { + const state = getAsyncHookFallbackState(promise, true); + + if (state[symbol]) { + if (state[symbol] !== value) { + // This can happen if a frozen promise created before hooks were enabled + // is used multiple times as a parent promise + // It's safe to ignore subsequent values + } + } else { + state[symbol] = value; + } +}; + +const getAsyncHookSymbolPromiseProtoDesc = (symbol, disallowGet) => ({ + set(value) { + if (Object.isExtensible(this)) { + Object.defineProperty(this, symbol, { + value, + writable: false, + configurable: false, + enumerable: false, + }); + } else { + // console.log('fallback set of async id', symbol, value, new Error().stack); + setAsyncIdFallback(this, symbol, value); + } + }, + get() { + if (disallowGet) { + return undefined; + } + const state = getAsyncHookFallbackState(this, false); + return state && state[symbol]; + }, + enumerable: false, + configurable: true, +}); + +export const setup = (withDestroy = true) => { + if (withDestroy) { + findAsyncSymbolsFromPromiseCreateHook(); + } else { + findAsyncSymbolsFromAsyncResource(); + } + + if (!Symbol.nodeAsyncHooksAsyncId || !Symbol.nodeAsyncHooksTriggerAsyncId) { + // console.log(`Async symbols not found, moving on`); + return; + } + + const PromiseProto = Promise.prototype; + Object.defineProperty( + PromiseProto, + Symbol.nodeAsyncHooksAsyncId, + getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksAsyncId), + ); + Object.defineProperty( + PromiseProto, + Symbol.nodeAsyncHooksTriggerAsyncId, + getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksTriggerAsyncId), + ); + + if (Symbol.nodeAsyncHooksDestroyed) { + Object.defineProperty( + PromiseProto, + Symbol.nodeAsyncHooksDestroyed, + getAsyncHookSymbolPromiseProtoDesc(Symbol.nodeAsyncHooksDestroyed, true), + ); + } else if (withDestroy) { + // console.warn(`Couldn't find destroyed symbol to setup trap`); + } +}; diff --git a/packages/init/pre.js b/packages/init/pre.js index 134a7a87a5..88af15e0f0 100644 --- a/packages/init/pre.js +++ b/packages/init/pre.js @@ -1,2 +1,4 @@ // Generic preamble for all shims. + +import './node-async_hooks-patch.js'; import '@endo/lockdown'; diff --git a/packages/init/test/test-async_hooks.js b/packages/init/test/test-async_hooks.js new file mode 100644 index 0000000000..9cf7ce64d6 --- /dev/null +++ b/packages/init/test/test-async_hooks.js @@ -0,0 +1,79 @@ +/* global globalThis, $262 */ + +import '../index.js'; +import test from 'ava'; +import { createHook } from 'async_hooks'; +import { setTimeout } from 'timers'; + +const gcP = (async () => { + let gc = globalThis.gc || (typeof $262 !== 'undefined' ? $262.gc : null); + if (!gc) { + gc = () => { + Array.from({ length: 2 ** 24 }, () => Math.random()); + }; + } + return gc; +})(); + +test('async_hooks Promise patch', async t => { + const hasSymbols = + Symbol.nodeAsyncHooksAsyncId && Symbol.nodeAsyncHooksTriggerAsyncId; + let resolve; + const q = (() => { + const p1 = new Promise(r => (resolve = r)); + t.deepEqual( + Reflect.ownKeys(p1), + [], + `Promise instances don't start with any own keys`, + ); + harden(p1); + + // The `.then()` fulfillment triggers the "before" hook for `p2`, + // which enforces that `p2` is a tracked promise by installing async id symbols + const p2 = Promise.resolve().then(() => {}); + t.deepEqual( + Reflect.ownKeys(p2), + [], + `Promise instances don't start with any own keys`, + ); + harden(p2); + + const testHooks = createHook({ + init() {}, + before() {}, + // after() {}, + destroy() {}, + }); + testHooks.enable(); + + // Create a promise with symbols attached + const p3 = Promise.resolve(); + if (hasSymbols) { + t.truthy(Reflect.ownKeys(p3)); + } + + return Promise.resolve().then(() => { + resolve(8); + // ret is a tracked promise created from parent `p1` + // async_hooks will attempt to get the asyncId from `p1` + // which was created and frozen before the symbols were installed + const ret = p1.then(() => {}); + // Trigger attempting to get asyncId of `p1` again, which in current + // node versions will fail and generate a new one because of an own check + p1.then(() => {}); + + if (hasSymbols) { + t.truthy(Reflect.ownKeys(ret)); + } + + // testHooks.disable(); + + return ret; + }); + })(); + + return q + .then(() => new Promise(r => setTimeout(r, 0, gcP))) + .then(gc => gc()) + .then(() => new Promise(r => setTimeout(r))); +}); diff --git a/packages/ses/src/whitelist.js b/packages/ses/src/whitelist.js index 3c21774213..05063699cc 100644 --- a/packages/ses/src/whitelist.js +++ b/packages/ses/src/whitelist.js @@ -501,6 +501,9 @@ export const whitelist = { keyFor: fn, match: 'symbol', matchAll: 'symbol', + nodeAsyncHooksAsyncId: 'symbol', + nodeAsyncHooksTriggerAsyncId: 'symbol', + nodeAsyncHooksDestroyed: 'symbol', prototype: '%SymbolPrototype%', replace: 'symbol', search: 'symbol', @@ -1289,6 +1292,10 @@ export const whitelist = { finally: fn, then: fn, '@@toStringTag': 'string', + // Non-standard, used in node to prevent async_hooks from breaking + '@@nodeAsyncHooksAsyncId': accessor, + '@@nodeAsyncHooksTriggerAsyncId': accessor, + '@@nodeAsyncHooksDestroyed': accessor, }, '%InertAsyncFunction%': {