feat(endomoat): support native modules in policy gen

packages/core/src/mergePolicy.js

@@ -5,23 +5,38 @@ const {
 } = require('lavamoat-tofu/src/util')
 const mergeDeep = require('merge-deep')
 
-module.exports = { mergePolicy }
-
+/**
+ * Merges two policies together.
+ *
+ * `policyB` overwrites `policyA` where concatenation is not possible
+ *
+ * @param {import('./schema').LavaMoatPolicy} policyA First policy
+ * @param {import('./schema').LavaMoatPolicy
+ *   | import('./schema').LavaMoatPolicyOverrides} [policyB]
+ *   Second policy or policy override
+ * @returns {import('./schema').LavaMoatPolicy} Merged policy or `policyA` if
+ *   `policyB` not provided
+ */
 function mergePolicy(policyA, policyB) {
-  const mergedPolicy = mergeDeep(policyA, policyB)
-  Object.values(mergedPolicy.resources).forEach((packagePolicy) => {
-    if ('globals' in packagePolicy) {
-      packagePolicy.globals = dedupePolicyPaths(packagePolicy.globals)
-    }
-    if ('builtin' in packagePolicy) {
-      packagePolicy.builtin = dedupePolicyPaths(packagePolicy.builtin)
-    }
-  })
-  return mergedPolicy
+  if (policyB) {
+    const mergedPolicy = mergeDeep(policyA, policyB)
+    Object.values(mergedPolicy.resources).forEach((packagePolicy) => {
+      if ('globals' in packagePolicy) {
+        packagePolicy.globals = dedupePolicyPaths(packagePolicy.globals)
+      }
+      if ('builtin' in packagePolicy) {
+        packagePolicy.builtin = dedupePolicyPaths(packagePolicy.builtin)
+      }
+    })
+    return /** @type {LavaMoatPolicy} */ (mergedPolicy)
+  }
+  return policyA
 }
 
 function dedupePolicyPaths(packagePolicy) {
   const itemMap = objToMap(packagePolicy)
   reduceToTopmostApiCalls(itemMap)
   return mapToObj(itemMap)
 }
+
+module.exports = { mergePolicy }

packages/core/src/schema/lavamoat-policy.v0-0-1.schema.ts

@@ -107,8 +107,9 @@ export interface BuiltinPolicy {
  * `true` to allow and `false` to deny
  */
 export interface PackagePolicy {
-  [k: string]: boolean
+  [k: string]: PackagePolicyValue
 }
+
 /**
  * Custom run-time module resolutions by direct dependency
  */
@@ -123,3 +124,7 @@ export interface Resolutions {
     [k: string]: string
   }
 }
+
+export type PackagePolicyValue = DynamicPkgPolicy | boolean
+
+export type DynamicPkgPolicy = 'dynamic'

packages/endomoat/package.json

@@ -36,11 +36,12 @@
   "scripts": {
     "lint:deps": "depcheck",
     "test": "npm run test:run",
-    "test:run": "ava"
+    "test:run": "ava",
+    "update-policy": "node ./scripts/update-own-policy.js"
   },
   "dependencies": {
     "@endo/compartment-mapper": "1.1.4",
-    "@endo/evasive-transform": "1.0.4",
+    "@endo/evasive-transform": "1.1.1",
     "@types/node": "18.19.28",
     "json-stable-stringify": "1.1.1",
     "lavamoat-core": "^15.3.0",

packages/endomoat/scripts/update-own-policy.js

@@ -0,0 +1,29 @@
+/**
+ * This script generates a policy for endomoat itself. This is needed to allow
+ * the default attenuator to execute.
+ *
+ * The result of this is merged into the user-provided policy (or policies)
+ * during conversion to an Endo policy. Because the Endo policy is not
+ * persisted, making changes in endomoat won't break existing policies.
+ *
+ * By persisting the policy to disk, we tradeoff initial execution time for
+ * needing to update the policy before release. Is this a good idea? I don't
+ * know.
+ *
+ * @packageDocumentation
+ */
+
+import { fileURLToPath } from 'node:url'
+import { generateAndWritePolicy } from '../src/index.js'
+
+const PHONY_ENTRYPOINT = new URL('./use-self/index.js', import.meta.url)
+
+const POLICY_OVERRIDE_PATH = fileURLToPath(
+  new URL('../src/policy-override.json', import.meta.url)
+)
+
+generateAndWritePolicy(PHONY_ENTRYPOINT, {
+  policyPath: POLICY_OVERRIDE_PATH,
+}).then(() => {
+  console.error('Wrote %s', POLICY_OVERRIDE_PATH)
+})

packages/endomoat/scripts/use-self/index.js

@@ -0,0 +1 @@
+import '@lavamoat/endomoat/attenuator'

packages/endomoat/scripts/use-self/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "use-self",
+  "version": "0.0.0",
+  "type": "module",
+  "main": "index.js",
+  "dependencies": {
+    "@lavamoat/endomoat": "file:../.."
+  }
+}

packages/endomoat/src/constants.js

@@ -36,11 +36,15 @@ export const POLICY_ITEM_WRITE = 'write'
  */
 export const POLICY_ITEM_WILDCARD = 'any'
 
+export const POLICY_ITEM_DYNAMIC = 'dynamic'
+
 /**
  * Designator for the root policy item in a LavaMoat policy
  */
 export const LAVAMOAT_PKG_POLICY_ROOT = '$root$'
 
+export const LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC = 'dynamic'
+
 /**
  * Name of the `packages` property of a `LavaMoatPackagePolicy`
  */
@@ -65,3 +69,13 @@ export const LMR_TYPE_BUILTIN = 'builtin'
  * `js` module type for a `LavamoatModuleRecord`
  */
 export const LMR_TYPE_SOURCE = 'js'
+
+/**
+ * `native` module type for a `LavamoatModuleRecord`
+ */
+export const LMR_TYPE_NATIVE = 'native'
+
+/**
+ * Name of Endo's `bytes` parser
+ */
+export const ENDO_PARSER_BYTES = 'bytes'

packages/endomoat/src/import-hook.js

@@ -1,3 +1,6 @@
+import { Module } from 'node:module'
+import { fileURLToPath } from 'node:url'
+
 const { freeze, keys, assign } = Object
 
 /**
@@ -19,3 +22,20 @@ export const importHook = async (specifier) => {
     })
   )
 }
+
+/** @type {import('@endo/compartment-mapper').DynamicImportHook} */
+export const importNowHook = (specifier, packageLocation) => {
+  const require = Module.createRequire(fileURLToPath(packageLocation))
+  /** @type {object} */
+  const ns = require(specifier)
+  return freeze(
+    /** @type {import('ses').ThirdPartyStaticModuleInterface} */ ({
+      imports: [],
+      exports: keys(ns),
+      execute: (moduleExports) => {
+        moduleExports.default = ns
+        assign(moduleExports, ns)
+      },
+    })
+  )
+}

packages/endomoat/src/index.js

@@ -17,8 +17,8 @@ lockdown({
 
 import { importLocation } from '@endo/compartment-mapper'
 import { pathToFileURL } from 'node:url'
-import { importHook } from './import-hook.js'
-import { moduleTransforms } from './module-transforms.js'
+import { importHook, importNowHook } from './import-hook.js'
+import { syncModuleTransforms } from './module-transforms.js'
 import { toEndoPolicy } from './policy-converter.js'
 import { generatePolicy } from './policy-gen/index.js'
 import { isPolicy } from './policy.js'
@@ -73,20 +73,30 @@ export async function run(entrypointPath, policyOrOpts = {}, opts = {}) {
     policy = await generatePolicy(entrypointPath, generateOpts)
   }
 
-  const endoPolicy = toEndoPolicy(policy)
+  const endoPolicy = await toEndoPolicy(policy)
   const readPowers = makeReadPowers(runOpts.readPowers)
 
   const url =
     entrypointPath instanceof URL
       ? `${entrypointPath}`
       : `${pathToFileURL(entrypointPath)}`
 
-  const { namespace } = await importLocation(readPowers, url, {
+  /**
+   * @type {import('@endo/compartment-mapper').SyncArchiveOptions &
+   *   import('@endo/compartment-mapper').ExecuteOptions}
+   */
+  const importOpts = {
     policy: endoPolicy,
     globals: globalThis,
     importHook,
-    moduleTransforms,
-  })
+    dynamicHook: importNowHook,
+    syncModuleTransforms,
+    fallbackLanguageForExtension: {
+      node: 'bytes',
+    },
+  }
+
+  const { namespace } = await importLocation(readPowers, url, importOpts)
 
   return namespace
 }

packages/endomoat/src/module-transforms.js

@@ -1,4 +1,4 @@
-import { evadeCensor } from '@endo/evasive-transform'
+import { evadeCensorSync } from '@endo/evasive-transform'
 import { applySourceTransforms } from 'lavamoat-core'
 
 export const decoder = new TextDecoder()
@@ -9,15 +9,15 @@ export const encoder = new TextEncoder()
  * restrictions
  *
  * @param {import('@endo/compartment-mapper').Language} parser
- * @returns {import('@endo/compartment-mapper').ModuleTransform}
+ * @returns {import('@endo/compartment-mapper').SyncModuleTransform}
  */
 export function createModuleTransform(parser) {
-  return async (sourceBytes, specifier, location, _packageLocation, opts) => {
+  return (sourceBytes, specifier, location, _packageLocation, opts) => {
     let source = decoder.decode(sourceBytes)
     // FIXME: this function calls stuff we could get in `ses/tools.js`
     // except `evadeDirectEvalExpressions`. unclear if we should be using this from `lavamoat-core`
     source = applySourceTransforms(source)
-    const { code, map } = await evadeCensor(source, {
+    const { code, map } = evadeCensorSync(source, {
       sourceMap: opts?.sourceMap,
       sourceUrl: new URL(specifier, location).href,
       sourceType: 'module',
@@ -30,7 +30,7 @@ export function createModuleTransform(parser) {
 /**
  * Standard set of module transforms for our purposes
  */
-export const moduleTransforms = /** @type {const} */ ({
+export const syncModuleTransforms = /** @type {const} */ ({
   cjs: createModuleTransform('cjs'),
   mjs: createModuleTransform('mjs'),
 })

packages/endomoat/src/policy-converter.js

@@ -1,11 +1,15 @@
+import { mergePolicy } from 'lavamoat-core'
 import {
   LAVAMOAT_PKG_POLICY_ROOT,
+  LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC,
+  POLICY_ITEM_DYNAMIC,
   POLICY_ITEM_ROOT,
   POLICY_ITEM_WILDCARD,
   RSRC_POLICY_BUILTINS,
   RSRC_POLICY_GLOBALS,
   RSRC_POLICY_PKGS,
 } from './constants.js'
+import { readPolicy } from './policy.js'
 
 const { isArray } = Array
 const { entries, fromEntries } = Object
@@ -40,6 +44,11 @@ function toEndoRsrcPkgsPolicyBuiltins(item) {
           'Expected a FullAttenuationDefinition; got a boolean'
         )
       }
+      if (itemForBuiltin === 'dynamic') {
+        throw new TypeError(
+          'Expected a FullAttenuationDefinition; got "dynamic"'
+        )
+      }
       if (isArray(itemForBuiltin)) {
         throw new TypeError(
           'Expected a FullAttenuationDefinition; got an array'
@@ -61,7 +70,8 @@ function toEndoRsrcPkgsPolicyBuiltins(item) {
  * Converts LavaMoat `ResourcePolicy.packages` to Endo's
  * `PackagePolicy.packages`
  *
- * @param {Record<string, boolean>} [item] - A value in `ResourcePolicy`
+ * @param {import('lavamoat-core').PackagePolicy} [item] - A value in
+ *   `ResourcePolicy`
  * @returns {import('./types.js').LavaMoatPackagePolicy['packages']}
  */
 function toEndoRsrcPkgsPolicyPkgs(item) {
@@ -78,7 +88,10 @@ function toEndoRsrcPkgsPolicyPkgs(item) {
     if (key === LAVAMOAT_PKG_POLICY_ROOT) {
       throw new TypeError('Unexpected root package policy')
     } else {
-      policyItem[key] = value
+      policyItem[key] =
+        value === LAVAMOAT_PKG_POLICY_VALUE_DYNAMIC
+          ? POLICY_ITEM_DYNAMIC
+          : Boolean(value)
     }
   }
   return policyItem
@@ -122,9 +135,9 @@ function toEndoRsrcPkgsPolicyGlobals(item) {
 function toEndoRsrcPkgsPolicy(resources) {
   /** @type {import('./types.js').LavaMoatPackagePolicy} */
   const pkgPolicy = {
-    packages: toEndoRsrcPkgsPolicyPkgs(resources.packages),
-    globals: toEndoRsrcPkgsPolicyGlobals(resources.globals),
-    builtins: toEndoRsrcPkgsPolicyBuiltins(resources.builtin),
+    [RSRC_POLICY_PKGS]: toEndoRsrcPkgsPolicyPkgs(resources.packages),
+    [RSRC_POLICY_GLOBALS]: toEndoRsrcPkgsPolicyGlobals(resources.globals),
+    [RSRC_POLICY_BUILTINS]: toEndoRsrcPkgsPolicyBuiltins(resources.builtin),
   }
   return pkgPolicy
 }
@@ -133,12 +146,18 @@ function toEndoRsrcPkgsPolicy(resources) {
  * Converts a LavaMoat policy to an Endo policy
  *
  * @param {import('lavamoat-core').LavaMoatPolicy} lmPolicy
- * @returns {import('./types.js').LavaMoatEndoPolicy}
+ * @returns {Promise<import('./types.js').LavaMoatEndoPolicy>}
  */
-export function toEndoPolicy(lmPolicy) {
+export async function toEndoPolicy(lmPolicy) {
+  // policy for self; needed for attenuator
+  const overrides = await readPolicy(
+    new URL('./policy-override.json', import.meta.url)
+  )
+
+  const finalLMPolicy = mergePolicy(lmPolicy, overrides)
+
   /** @type {import('./types.js').LavaMoatEndoPolicy} */
   const endoPolicy = {
-    //TODO: generate a policy resource for the default attenuator
     defaultAttenuator: DEFAULT_ATTENUATOR,
     entry: {
       [RSRC_POLICY_GLOBALS]: [POLICY_ITEM_ROOT],
@@ -147,18 +166,12 @@ export function toEndoPolicy(lmPolicy) {
       noGlobalFreeze: true,
     },
     resources: fromEntries(
-      entries(lmPolicy.resources ?? {}).map(([rsrcName, rsrcPolicy]) => [
+      entries(finalLMPolicy.resources ?? {}).map(([rsrcName, rsrcPolicy]) => [
         rsrcName,
         toEndoRsrcPkgsPolicy(rsrcPolicy),
       ])
     ),
   }
-  // add this to make endo allow the attenuator at all, TODO: generate this from the policy or build into Endo
-  endoPolicy.resources['@lavamoat/endomoat'] = {
-    [RSRC_POLICY_PKGS]: POLICY_ITEM_WILDCARD,
-    [RSRC_POLICY_GLOBALS]: POLICY_ITEM_WILDCARD,
-    [RSRC_POLICY_BUILTINS]: POLICY_ITEM_WILDCARD,
-  }
 
   return endoPolicy
 }