.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: jspm # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/build.yml

@@ -0,0 +1,69 @@
+name: Build & Test
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+env:
+  WASI_VERSION: 12
+  WASI_VERSION_FULL: "12.0"
+  EMCC_VERSION: "1.40.1-fastcomp"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, windows-2016]
+
+    steps:
+      - name: Check out Git repository
+        uses: actions/checkout@v2
+
+      - name: Install wasi-sdk
+        shell: bash
+        run: |
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            export WASI_OS="linux";
+          fi
+          if [[ "$RUNNER_OS" == "macOS" ]]; then
+            export WASI_OS="macos";
+          fi
+          if [[ "$RUNNER_OS" == "Windows" ]]; then
+            export WASI_OS="mingw";
+          fi
+
+          curl -sL https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_VERSION}/wasi-sdk-${WASI_VERSION_FULL}-${WASI_OS}.tar.gz -O
+          # check if package downloaded
+          ls -la
+          tar xvf wasi-sdk-${WASI_VERSION_FULL}-${WASI_OS}.tar.gz
+
+          # print clang version
+          ./wasi-sdk-${WASI_VERSION_FULL}/bin/clang --version
+
+      - name: Install Emscripten
+        run: |
+          git clone https://github.com/emscripten-core/emsdk.git
+          cd ./emsdk
+          ./emsdk install ${{ env.EMCC_VERSION }}
+          ./emsdk activate ${{ env.EMCC_VERSION }}
+          ./fastcomp/emscripten/emcc -v
+
+      - name: npm Install
+        run: npm install
+
+      - name: Install Chomp
+        run: cargo install chompbuild
+
+      - name: Compile to Wasm & Test Wasm
+        env:
+          WASI_PATH: './wasi-sdk-${{ env.WASI_VERSION_FULL }}'
+          EMSDK_PATH: './emsdk'
+          EMSCRIPTEN_BIN: './emsdk/fastcomp/emscripten/emcc'
+        run: chomp test
+
+      - name: Benchmark Wasm
+        run: chomp bench

.gitignore

@@ -1,6 +1,6 @@
 node_modules
 lib/lexer.wat
-lib/lexer.js
 dist
 package-lock.json
 yarn.lock
+lib/lexer.emcc.js

README.md

@@ -1,6 +1,6 @@
 # ES Module Lexer
 
-[![Build Status][travis-image]][travis-url]
+[![Build Status][actions-image]][actions-url]
 
 A JS module syntax lexer used in [es-module-shims](https://github.com/guybedford/es-module-shims).
 
@@ -42,22 +42,23 @@ import { init, parse } from 'es-module-lexer';
   await init;
 
   const source = `
-    import { name } from 'mod';
+    import { name } from 'mod\\u1011';
     import json from './json.json' assert { type: 'json' }
     export var p = 5;
     export function q () {
 
     };
 
     // Comments provided to demonstrate edge cases
-    import /*comment!*/ ('asdf', { assert: { type: 'json' }});
+    import /*comment!*/ (  'asdf', { assert: { type: 'json' }});
     import /*comment!*/.meta.asdf;
   `;
 
   const [imports, exports] = parse(source, 'optional-sourcename');
 
-  // Returns "mod"
+  // Returns "modထ"
   imports[0].n
+  // Returns "mod\u1011"
   source.substring(imports[0].s, imports[0].e);
   // "s" = start
   // "e" = end
@@ -69,34 +70,41 @@ import { init, parse } from 'es-module-lexer';
 
   // Returns "{ type: 'json' }"
   source.substring(imports[1].a, imports[1].se);
-  // "a" = assert
+  // "a" = assert, -1 for no assertion
 
   // Returns "p,q"
   exports.toString();
 
   // Dynamic imports are indicated by imports[2].d > -1
-  // In this case the "d" index is the start of the dynamic import
+  // In this case the "d" index is the start of the dynamic import bracket
   // Returns true
   imports[2].d > -1;
 
-  // Returns "asdf"
+  // Returns "asdf" (only for string literal dynamic imports)
   imports[2].n
+  // Returns "import /*comment!*/ (  'asdf', { assert: { type: 'json' } })"
+  source.substring(imports[2].ss, imports[2].se);
   // Returns "'asdf'"
   source.substring(imports[2].s, imports[2].e);
-  // Returns "import /*comment!*/ ("
-  source.substring(imports[2].d, imports[2].s);
-  // Returns "import /*comment!*/ ('asdf', { assert: { type: 'json' } })"
-  source.substring(imports[2].d, imports[2].se + 1);
+  // Returns "(  'asdf', { assert: { type: 'json' } })"
+  source.substring(imports[2].d, imports[2].se);
   // Returns "{ assert: { type: 'json' } }"
-  source.substring(imports[2].a, imports[2].e);
-  // ss is the same as d
-  // as, ae not used for dynamic imports
+  source.substring(imports[2].a, imports[2].se - 1);
+
+  // For non-string dynamic import expressions:
+  // - n will be undefined
+  // - a is currently -1 even if there is an assertion
+  // - e is currently the character before the closing )
+
+  // For nested dynamic imports, the se value of the outer import is -1 as end tracking does not
+  // currently support nested dynamic immports
 
   // import.meta is indicated by imports[2].d === -2
   // Returns true
   imports[2].d === -2;
   // Returns "import /*comment!*/.meta"
   source.substring(imports[2].s, imports[2].e);
+  // ss and se are the same for import meta
 })();
 ```
 
@@ -232,20 +240,38 @@ test/samples/*.js (3123 KiB)
 
 ### Building
 
-To build download the WASI SDK from https://github.com/WebAssembly/wasi-sdk/releases.
+This project uses [Chomp](https://chompbuild.com) for building.
+
+With Chomp installed, download the WASI SDK 12.0 from https://github.com/WebAssembly/wasi-sdk/releases/tag/wasi-sdk-12.
+
+- [Linux](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz)
+- [Windows (MinGW)](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-mingw.tar.gz)
+- [macOS](https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-macos.tar.gz)
 
-The Makefile assumes the existence of "wasi-sdk-11.0" and "wabt" (optional) as sibling folders to this project.
+Locate the WASI-SDK as a sibling folder, or customize the path via the `WASI_PATH` environment variable.
 
-The build through the Makefile is then run via `make lib/lexer.wasm`, which can also be triggered via `npm run build:wasm` to create `dist/lexer.js`.
+Emscripten emsdk is also assumed to be a sibling folder or via the `EMSDK_PATH` environment variable.
 
-On Windows it may be preferable to use the Linux subsystem.
+Example setup:
+
+```
+git clone https://github.com:guybedford/es-module-lexer
+git clone https://github.com/emscripten-core/emsdk
+wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-12/wasi-sdk-12.0-linux.tar.gz
+gunzip wasi-sdk-12.0-linux.tar.gz
+tar -xf wasi-sdk-12.0-linux.tar
+mv wasi-sdk-12.0-linux.tar wasi-sdk-12.0
+cargo install chompbuild
+cd es-module-lexer
+chomp test
+```
 
-After the Web Assembly build, the CJS build can be triggered via `npm run build`.
+For the `asm.js` build, git clone `emsdk` from  is assumed to be a sibling folder as well.
 
 ### License
 
 MIT
 
-[travis-url]: https://travis-ci.org/guybedford/es-module-lexer
-[travis-image]: https://travis-ci.org/guybedford/es-module-lexer.svg?branch=master
+[actions-image]: https://github.com/guybedford/es-module-lexer/actions/workflows/build.yml/badge.svg
+[actions-url]: https://github.com/guybedford/es-module-lexer/actions/workflows/build.yml
 

bench/index.js

@@ -27,27 +27,31 @@ Promise.resolve().then(async () => {
 		return Math.round(Number(end - start) / 1e6);
 	}
 
-	console.log('--- JS Build ---');
-	console.log('Module load time');
-	{
-		const start = process.hrtime.bigint();
-		var { parse } = await import('../dist/lexer.asm.js');
-		console.log(`> ${c.bold.green(Math.round(Number(process.hrtime.bigint() - start) / 1e6) + 'ms')}`);
+	if (!process.env.BENCH || process.env.BENCH === 'js') {
+		console.log('--- JS Build ---');
+		console.log('Module load time');
+		{
+			const start = process.hrtime.bigint();
+			var { parse } = await import('../dist/lexer.asm.js');
+			console.log(`> ${c.bold.green(Math.round(Number(process.hrtime.bigint() - start) / 1e6) + 'ms')}`);
+		}
+
+		doRun();
 	}
-
-	doRun();
-
-	console.log('--- Wasm Build ---');
-	console.log('Module load time');
-	{
-		const start = process.hrtime.bigint();
-		var { parse, init } = await import('../dist/lexer.js');
-		await init;
-		console.log(`> ${c.bold.green(Math.round(Number(process.hrtime.bigint() - start) / 1e6) + 'ms')}`);
+
+	if (!process.env.BENCH || process.env.BENCH === 'wasm') {
+		console.log('--- Wasm Build ---');
+		console.log('Module load time');
+		{
+			const start = process.hrtime.bigint();
+			var { parse, init } = await import('../dist/lexer.js');
+			await init;
+			console.log(`> ${c.bold.green(Math.round(Number(process.hrtime.bigint() - start) / 1e6) + 'ms')}`);
+		}
+
+		doRun();
 	}
 
-	doRun();
-
 	function doRun () {
 		console.log('Cold Run, All Samples');
 		let totalSize = 0;