feat(endomoat): support native modules in policy generation [ci skip]

packages/endomoat/src/constants.js

@@ -36,11 +36,15 @@ export const POLICY_ITEM_WRITE = 'write'
  */
 export const POLICY_ITEM_WILDCARD = 'any'
 
+export const POLICY_ITEM_DYNAMIC = 'dynamic'
+
 /**
  * Designator for the root policy item in a LavaMoat policy
  */
 export const LAVAMOAT_PKG_POLICY_ROOT = '$root$'
 
+export const LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC = 'dynamic'
+
 /**
  * Name of the `packages` property of a `LavaMoatPackagePolicy`
  */
@@ -65,3 +69,13 @@ export const LMR_TYPE_BUILTIN = 'builtin'
  * `js` module type for a `LavamoatModuleRecord`
  */
 export const LMR_TYPE_SOURCE = 'js'
+
+/**
+ * `native` module type for a `LavamoatModuleRecord`
+ */
+export const LMR_TYPE_NATIVE = 'native'
+
+/**
+ * Name of Endo's `bytes` parser
+ */
+export const ENDO_PARSER_BYTES = 'bytes'

packages/endomoat/src/import-hook.js

@@ -1,3 +1,6 @@
+import { Module } from 'node:module'
+import { fileURLToPath } from 'node:url'
+
 const { freeze, keys, assign } = Object
 
 /**
@@ -19,3 +22,20 @@ export const importHook = async (specifier) => {
     })
   )
 }
+
+/** @type {import('@endo/compartment-mapper').DynamicImportHook} */
+export const importNowHook = (specifier, packageLocation) => {
+  const require = Module.createRequire(fileURLToPath(packageLocation))
+  /** @type {object} */
+  const ns = require(specifier)
+  return freeze(
+    /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({
+      imports: [],
+      exports: keys(ns),
+      execute: (moduleExports) => {
+        moduleExports.default = ns
+        assign(moduleExports, ns)
+      },
+    })
+  )
+}

packages/endomoat/src/index.js

@@ -17,7 +17,7 @@ lockdown({
 
 import { importLocation } from '@endo/compartment-mapper'
 import { pathToFileURL } from 'node:url'
-import { importHook } from './import-hook.js'
+import { importHook, importNowHook } from './import-hook.js'
 import { moduleTransforms } from './module-transforms.js'
 import { toEndoPolicy } from './policy-converter.js'
 import { generatePolicy } from './policy-gen/index.js'
@@ -85,7 +85,11 @@ export async function run(entrypointPath, policyOrOpts = {}, opts = {}) {
     policy: endoPolicy,
     globals: globalThis,
     importHook,
+    dynamicHook: importNowHook,
     moduleTransforms,
+    fallbackLanguageForExtension: {
+      node: 'bytes',
+    },
   })
 
   return namespace

packages/endomoat/src/policy-converter.js

@@ -1,5 +1,7 @@
 import {
   LAVAMOAT_PKG_POLICY_ROOT,
+  LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC,
+  POLICY_ITEM_DYNAMIC,
   POLICY_ITEM_ROOT,
   POLICY_ITEM_WILDCARD,
   RSRC_POLICY_BUILTINS,
@@ -40,6 +42,11 @@ function toEndoRsrcPkgsPolicyBuiltins(item) {
           'Expected a FullAttenuationDefinition; got a boolean'
         )
       }
+      if (itemForBuiltin === 'dynamic') {
+        throw new TypeError(
+          'Expected a FullAttenuationDefinition; got "dynamic"'
+        )
+      }
       if (isArray(itemForBuiltin)) {
         throw new TypeError(
           'Expected a FullAttenuationDefinition; got an array'
@@ -78,7 +85,10 @@ function toEndoRsrcPkgsPolicyPkgs(item) {
     if (key === LAVAMOAT_PKG_POLICY_ROOT) {
       throw new TypeError('Unexpected root package policy')
     } else {
-      policyItem[key] = value
+      policyItem[key] =
+        value === LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC
+          ? POLICY_ITEM_DYNAMIC
+          : Boolean(value)
     }
   }
   return policyItem

packages/endomoat/src/policy-gen/index.js

@@ -180,6 +180,9 @@ export async function loadCompartmentMap(
       moduleLocation,
       importHook,
       moduleTransforms,
+      fallbackLanguageForExtension: {
+        node: 'bytes',
+      },
     })
 
   // `compartmentRenames` is a mapping of filepath to compartment name;

packages/endomoat/src/policy-gen/policy-generator-context.js

@@ -8,8 +8,10 @@ import { isBuiltin as nodeIsBuiltin } from 'node:module'
 import path from 'node:path'
 import { fileURLToPath } from 'node:url'
 import {
+  ENDO_PARSER_BYTES,
   LAVAMOAT_PKG_POLICY_ROOT,
   LMR_TYPE_BUILTIN,
+  LMR_TYPE_NATIVE,
   LMR_TYPE_SOURCE,
 } from '../constants.js'
 import { defaultReadPowers } from '../power.js'
@@ -289,7 +291,10 @@ export class PolicyGeneratorContext {
    * @returns {Promise<import('lavamoat-core').LavamoatModuleRecord[]>}
    * @internal
    */
-  async buildModuleRecordsForSource(specifier, { record, sourceLocation }) {
+  async buildModuleRecordsForSource(
+    specifier,
+    { parser, record, sourceLocation }
+  ) {
     if (!sourceLocation) {
       // XXX: why would we not have a sourceLocation?
       throw new TypeError(
@@ -317,17 +322,24 @@ export class PolicyGeneratorContext {
      * The `ModuleSource.content` prop is already pre-processed by Endo, and we
      * do not want that, since it befouls our AST crawling.
      *
+     * This will not be run if the `parser` is `bytes`.
+     *
      * @remarks
      * Doing this first since it may be more likely to fail than the other
-     * operations below
-     * @type {NonNullable<
-     *   import('lavamoat-core').LavamoatModuleRecord['content']
-     * >}
+     * operations below.
+     * @type {string | undefined}
      * @todo Modify Endo to surface the original source
+     *
+     * @todo Add more exceptions to the parsers?
      */
-    const content = await this.#readPowers
-      .read(sourceLocation)
-      .then((buffer) => PolicyGeneratorContext.#decoder.decode(buffer))
+    let content
+
+    await Promise.resolve()
+    if (parser !== ENDO_PARSER_BYTES) {
+      content = await this.#readPowers
+        .read(sourceLocation)
+        .then((buffer) => PolicyGeneratorContext.#decoder.decode(buffer))
+    }
 
     /**
      * The {@link LavamoatModuleRecord.file} prop
@@ -356,7 +368,7 @@ export class PolicyGeneratorContext {
         packageName: this.packageName,
         importMap,
         content,
-        type: LMR_TYPE_SOURCE,
+        type: parser === ENDO_PARSER_BYTES ? LMR_TYPE_NATIVE : LMR_TYPE_SOURCE,
       }),
     ]
 

packages/endomoat/test/fixture/dynamic/README.md

@@ -0,0 +1,5 @@
+This fixture is not used directly; it's only here in order to create a snapshot:
+
+```bash
+npm run start
+```

packages/endomoat/test/fixture/dynamic/index.js

@@ -0,0 +1,2 @@
+import { hello as otherHello } from 'dynamic-require'
+export const hello = 'hello ' + otherHello

packages/endomoat/test/fixture/dynamic/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "hello",
+  "version": "0.0.0",
+  "type": "module",
+  "description": "this code actually runs",
+  "private": true,
+  "main": "index.js",
+  "scripts": {
+    "start": "npx snapshot-fs . ../json/dynamic.json"
+  },
+  "dependencies": {
+    "dummy": "0.0.0",
+    "dynamic-require": "0.0.0"
+  }
+}

packages/endomoat/test/fixture/json/dynamic.json

@@ -0,0 +1,12 @@
+{
+  "/package.json": "{\n  \"name\": \"hello\",\n  \"version\": \"0.0.0\",\n  \"type\": \"module\",\n  \"description\": \"this code actually runs\",\n  \"private\": true,\n  \"main\": \"index.js\",\n  \"scripts\": {\n    \"start\": \"npx snapshot-fs . ../json/dynamic.json\"\n  },\n  \"dependencies\": {\n    \"dummy\": \"0.0.0\",\n    \"dynamic-require\": \"0.0.0\"\n  }\n}\n",
+  "/index.js": "import { hello as otherHello } from 'dynamic-require'\nexport const hello = 'hello ' + otherHello\n",
+  "/README.md": "This fixture is not used directly; it's only here in order to create a snapshot:\n\n```bash\nnpm run start\n```\n",
+  "/node_modules/muddy/package.json": "{\n  \"name\": \"muddy\",\n  \"version\": \"0.0.0\"\n}\n",
+  "/node_modules/muddy/index.js": "module.exports = \"world\"\n",
+  "/node_modules/dynamic-require/world.js": "module.exports = 'world'\n",
+  "/node_modules/dynamic-require/package.json": "{\n  \"name\": \"dynamic-require\",\n  \"version\": \"1.0.0\",\n  \"license\": \"ISC\",\n  \"private\": true,\n  \"main\": \"index.js\",\n  \"dependencies\": {\n\n  }\n}\n",
+  "/node_modules/dynamic-require/index.js": "function dynamic (value) {\n  return require(value)\n}\n\nexports.hello = dynamic('dummy')\n",
+  "/node_modules/dummy/package.json": "{\n  \"name\": \"dummy\",\n  \"version\": \"0.0.0\",\n  \"dependencies\": {\"muddy\": \"0.0.0\"}\n}\n",
+  "/node_modules/dummy/index.js": "module.exports = require('muddy')\n"
+}

packages/endomoat/test/fixture/json/phony-native.json

@@ -0,0 +1,8 @@
+{
+  "/package.json": "{\n  \"name\": \"native\",\n  \"version\": \"1.0.0\",\n  \"type\": \"module\",\n  \"license\": \"ISC\",\n  \"private\": true,\n  \"main\": \"app.js\",\n  \"dependencies\": {\n    \"hello\": \"^1.0.0\"\n  }\n}\n",
+  "/index.js": "import addon from 'hello'\nexport const hello = addon.hello()\n",
+  "/node_modules/hello/package.json": "{\n  \"name\": \"hello\",\n  \"version\": \"0.0.0\",\n  \"description\": \"this code does not run\",\n  \"private\": true,\n  \"main\": \"index.js\"\n}\n",
+  "/node_modules/hello/index.js": "var addon = require('./hello')\n\nconsole.log(addon.hello()) // 'world'\n\nmodule.exports = addon\n",
+  "/node_modules/hello/hello.node": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000",
+  "/node_modules/hello/README.md": "This fixture is an example of a package shipping a native module (`hello.node`). `hello.node` is not _actually_ a native module, but it has the right extension and is full of null bytes. Good enough.\n"
+}

packages/endomoat/test/fixture/native/README.md

@@ -0,0 +1,5 @@
+This fixture is not used directly; it's only here in order to create a snapshot:
+
+```bash
+npm run start
+```

packages/endomoat/test/fixture/native/index.js

@@ -0,0 +1,2 @@
+import addon from 'hello_world'
+export const hello = addon.hello()