Support building for externally shared js builtins

.github/workflows/build.yml

@@ -2,79 +2,72 @@ name: Test
 
 on:
   push:
-    branches: main
   pull_request:
     branches: main
 
-env:
-  WASI_VERSION: 12
-  WASI_VERSION_FULL: "12.0"
-  WABT_VERSION: "1.0.24"
-  EMCC_VERSION: "1.40.1-fastcomp"
-
 jobs:
   build:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    env:
+      WASI_SDK_VERSION: "21"
+      WASI_SDK_PATH: /opt/wasi-sdk
+      WABT_VERSION: "1.0.34"
+      WABT_PATH: /opt/wabt
+      BINARYEN_VERSION: "117"
+      BINARYEN_PATH: /opt/binaryen
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Prepare
-        id: preparation
-        run: |
-          export PWD=$(pwd);
-          echo "::set-output name=PROJ_ROOT::$PWD";
+        uses: actions/checkout@v4
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          cache: 'npm'
 
       - name: Install
-        run: npm install
+        run: npm ci
 
-      - name: Install wasi-sdk
-        env:
-          PROJ_ROOT: ${{ steps.preparation.outputs.PROJ_ROOT }}
+      - name: "Restore WASI SDK"
+        id: cache-wasi-sdk
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.WASI_SDK_PATH }}
+          key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}
+      - name: "Install WASI SDK"
+        if: steps.cache-wasi-sdk.outputs.cache-hit != 'true'
+        run: |
+          mkdir ${{ env.WASI_SDK_PATH }} && \
+          curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \
+          tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip
+      - name: "Restore WABT"
+        id: cache-wabt
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.WABT_PATH }}
+          key: ${{ runner.os }}-wabt-${{ env.WABT_VERSION }}
+      - name: "Install WABT"
+        if: steps.cache-wabt.outputs.cache-hit != 'true'
         run: |
-          cd $PROJ_ROOT;
-          cd ../;
-          export WASI_OS="linux"
-          curl -sL https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-${WASI_OS}.tar.gz -O
-          # check if package downloaded
-          ls -la
-          tar xvf wasi-sdk-${WASI_VERSION_FULL}-${WASI_OS}.tar.gz
-          # print clang version
-          ./wasi-sdk-${WASI_VERSION_FULL}/bin/clang --version
-      - name: Install wabt
-        env:
-          PROJ_ROOT: ${{ steps.preparation.outputs.PROJ_ROOT }}
+          mkdir ${{ env.WABT_PATH }} && \
+          curl -s -S --location https://github.com/WebAssembly/wabt/releases/download/${{ env.WABT_VERSION }}/wabt-${{ env.WABT_VERSION }}-ubuntu.tar.gz | \
+          tar --strip-components 1 --directory ${{ env.WABT_PATH }} --extract --gunzip
+      - name: "Restore Binaryen"
+        id: cache-binaryen
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.BINARYEN_PATH }}
+          key: ${{ runner.os }}-binaryen-${{ env.BINARYEN_VERSION }}
+      - name: "Install Binaryen"
+        if: steps.cache-binaryen.outputs.cache-hit != 'true'
         run: |
-          cd $PROJ_ROOT;
-          cd ../;
-          if [[ "$RUNNER_OS" == "Linux" ]]; then
-            export WABT_OS="ubuntu";
-          fi
-          if [[ "$RUNNER_OS" == "macOS" ]]; then
-            export WABT_OS="macos";
-          fi
-          if [[ "$RUNNER_OS" == "Windows" ]]; then
-            export WABT_OS="windows";
-          fi
-          curl -sL https://github.com/WebAssembly/wabt/releases/download/${WABT_VERSION}/wabt-${WABT_VERSION}-${WABT_OS}.tar.gz -O
-          # check if package downloaded
-          ls -la
-          tar xvf wabt-${WABT_VERSION}-${WABT_OS}.tar.gz
-          # check if wabt binaries installed
-          ls -la ./wabt-${WABT_VERSION}/bin/
+          mkdir ${{ env.BINARYEN_PATH }} && \
+          curl -s -S --location https://github.com/WebAssembly/binaryen/releases/download/version_${{ env.BINARYEN_VERSION }}/binaryen-version_${{ env.BINARYEN_VERSION }}-x86_64-linux.tar.gz | \
+          tar --strip-components 1 --directory ${{ env.WABT_PATH }} --extract --gunzip
       - name: Compile to Wasm & Test Wasm
-        env:
-          PROJ_ROOT: ${{ steps.preparation.outputs.PROJ_ROOT }}
         run: |
-          cd $PROJ_ROOT;
           rm lib/lexer.wasm;
           npm run build-wasm;
           npm run build;
           # test
           npm run test;
       - name: Benchmark Wasm
-        env:
-          PROJ_ROOT: ${{ steps.preparation.outputs.PROJ_ROOT }}
-        run: |
-          cd $PROJ_ROOT;
-          npm run bench;
+        run: npm run bench;

Makefile

@@ -1,10 +1,26 @@
 # These flags depend on the system and may be overridden
+
+ifeq ($(WASI_SDK_PATH),)
 WASM_CC := ../wasi-sdk-12.0/bin/clang
 WASM_CFLAGS := --sysroot=../wasi-sdk-12.0/share/wasi-sysroot
+else
+WASM_CC := $(WASI_SDK_PATH)/bin/clang
+WASM_CFLAGS := --sysroot=$(WASI_SDK_PATH)/share/wasi-sysroot
+endif
+
 WASM_LDFLAGS := -nostartfiles
 
+ifeq ($(WABT_PATH),)
 WASM2WAT := ../wabt/bin/wasm2wat
+else
+WASM2WAT := $(WABT_PATH)/bin/wasm2wat
+endif
+
+ifeq ($(BINARYEN_PATH),)
 WASM_OPT := ../binaryen/bin/wasm-opt
+else
+WASM_OPT := $(BINARYEN_PATH)/bin/wasm-opt
+endif
 
 # These are project-specific and are expected to be kept intact
 WASM_EXTRA_CFLAGS := -I include-wasm/ -Wno-logical-op-parentheses -Wno-parentheses -Oz

build.js

@@ -1,25 +1,72 @@
 const fs = require('fs');
-const terser = require('terser');
+const { buildSync } = require('esbuild');
 
-const MINIFY = true;
+const { EXTERNAL_PATH } = process.env;
+const MINIFY = !EXTERNAL_PATH;
 
 try { fs.mkdirSync('./dist'); }
 catch (e) {}
 
 const wasmBuffer = fs.readFileSync('./lib/lexer.wasm');
-const jsSource = fs.readFileSync('./src/lexer.js').toString();
 const pjson = JSON.parse(fs.readFileSync('./package.json').toString());
 
-const jsSourceProcessed = jsSource.replace('WASM_BINARY', wasmBuffer.toString('base64'));
+buildSync({
+  entryPoints: ['./src/lexer.js'],
+  outfile: './dist/lexer.mjs',
+  bundle: true,
+  minify: MINIFY,
+  platform: 'node',
+  format: 'esm',
+  banner: {
+    js: `/* cjs-module-lexer ${pjson.version} */`
+  },
+  define: EXTERNAL_PATH ? {
+    WASM_BINARY: 'undefined',
+    EXTERNAL_PATH: `'${EXTERNAL_PATH}'`,
+  } : {
+    WASM_BINARY: `'${wasmBuffer.toString('base64')}'`,
+    EXTERNAL_PATH: 'undefined'
+  }
+})
 
-const minified = MINIFY && terser.minify(jsSourceProcessed, {
-  module: true,
-  output: {
-    preamble: `/* cjs-module-lexer ${pjson.version} */`
+if (EXTERNAL_PATH) {
+  buildSync({
+    stdin: {
+      contents: `'use strict';
+let lazy;
+async function init () {
+  if (!lazy) {
+    lazy = await import(require('node:url').pathToFileURL(require('node:module').createRequire('${EXTERNAL_PATH}/dist/lexer.js').resolve('./lexer.mjs')));
   }
-});
+  module.exports = lazy;
+  return lazy.init();
+}
+
+function parse (source, name = '@') {
+  if (!lazy)
+    throw new Error('Not initialized');
+
+  return lazy.parse(source, name);
+}
 
-if (minified.error)
-  throw minified.error;
+module.exports = { init, parse };`,
+      loader: 'js',
+    },
+    outfile: './dist/lexer.js',
+    minify: MINIFY,
+    platform: 'node',
+    format: 'cjs',
+  });
+} else {
+  buildSync({
+    entryPoints: ['./dist/lexer.mjs'],
+    outfile: './dist/lexer.js',
+    minify: MINIFY,
+    platform: 'node',
+    format: 'cjs',
+    banner: {
+      js: `/* cjs-module-lexer ${pjson.version} */`
+    }
+  })
+}
 
-fs.writeFileSync('./dist/lexer.mjs', minified ? minified.code : jsSourceProcessed);