fix #2424: allow binary in transform and build

CHANGELOG.md

@@ -2,6 +2,33 @@
 
 ## Unreleased
 
+* Allow binary data as input to the JS `transform` and `build` APIs ([#2424](https://github.com/evanw/esbuild/issues/2424))
+
+    Previously esbuild's `transform` and `build` APIs could only take a string. However, some people want to use esbuild to convert binary data to base64 text. This is problematic because JavaScript strings represent UTF-16 text and esbuild internally operates on arrays of bytes, so all strings coming from JavaScript undergo UTF-16 to UTF-8 conversion before use. This meant that using esbuild in this way was doing base64 encoding of the UTF-8 encoding of the text, which was undesired.
+
+    With this release, esbuild now accepts `Uint8Array` in addition to string as an input format for the `transform` and `build` APIs. Now you can use esbuild to convert binary data to base64 text:
+
+    ```js
+    // Original code
+    import esbuild from 'esbuild'
+    console.log([
+      (await esbuild.transform('\xFF', { loader: 'base64' })).code,
+      (await esbuild.build({ stdin: { contents: '\xFF', loader: 'base64' }, write: false })).outputFiles[0].text,
+    ])
+    console.log([
+      (await esbuild.transform(new Uint8Array([0xFF]), { loader: 'base64' })).code,
+      (await esbuild.build({ stdin: { contents: new Uint8Array([0xFF]), loader: 'base64' }, write: false })).outputFiles[0].text,
+    ])
+
+    // Old output
+    [ 'module.exports = "w78=";\n', 'module.exports = "w78=";\n' ]
+    /* ERROR: The input to "transform" must be a string */
+
+    // New output
+    [ 'module.exports = "w78=";\n', 'module.exports = "w78=";\n' ]
+    [ 'module.exports = "/w==";\n', 'module.exports = "/w==";\n' ]
+    ```
+
 * Update the getter for `text` in build results ([#2423](https://github.com/evanw/esbuild/issues/2423))
 
     Output files in build results returned from esbuild's JavaScript API have both a `contents` and a `text` property to return the contents of the output file. The `contents` property is a binary UTF-8 Uint8Array and the `text` property is a JavaScript UTF-16 string. The `text` property is a getter that does the UTF-8 to UTF-16 conversion only if it's needed for better performance.

cmd/esbuild/service.go

@@ -435,11 +435,11 @@ func (service *serviceType) handleBuildRequest(id uint32, request map[string]int
 	}
 
 	// Optionally allow input from the stdin channel
-	if stdin, ok := request["stdinContents"].(string); ok {
+	if stdin, ok := request["stdinContents"].([]byte); ok {
 		if options.Stdin == nil {
 			options.Stdin = &api.StdinOptions{}
 		}
-		options.Stdin.Contents = stdin
+		options.Stdin.Contents = string(stdin)
 		if resolveDir, ok := request["stdinResolveDir"].(string); ok {
 			options.Stdin.ResolveDir = resolveDir
 		}
@@ -920,7 +920,7 @@ func (service *serviceType) convertPlugins(key int, jsPlugins interface{}, activ
 
 func (service *serviceType) handleTransformRequest(id uint32, request map[string]interface{}) []byte {
 	inputFS := request["inputFS"].(bool)
-	input := request["input"].(string)
+	input := string(request["input"].([]byte))
 	flags := decodeStringArray(request["flags"].([]interface{}))
 
 	options, err := cli.ParseTransformOptions(flags)

lib/shared/common.ts

@@ -210,7 +210,7 @@ function flagsForBuildOptions(
   entries: [string, string][],
   flags: string[],
   write: boolean,
-  stdinContents: string | null,
+  stdinContents: Uint8Array | null,
   stdinResolveDir: string | null,
   absWorkingDir: string | undefined,
   incremental: boolean,
@@ -221,7 +221,7 @@ function flagsForBuildOptions(
   let flags: string[] = [];
   let entries: [string, string][] = [];
   let keys: OptionKeys = Object.create(null);
-  let stdinContents: string | null = null;
+  let stdinContents: Uint8Array | null = null;
   let stdinResolveDir: string | null = null;
   let watchMode: types.WatchMode | null = null;
   pushLogFlags(flags, options, keys, isTTY, logLevelDefault);
@@ -354,7 +354,7 @@ function flagsForBuildOptions(
 
   if (stdin) {
     let stdinKeys: OptionKeys = Object.create(null);
-    let contents = getFlag(stdin, stdinKeys, 'contents', mustBeString);
+    let contents = getFlag(stdin, stdinKeys, 'contents', mustBeStringOrUint8Array);
     let resolveDir = getFlag(stdin, stdinKeys, 'resolveDir', mustBeString);
     let sourcefile = getFlag(stdin, stdinKeys, 'sourcefile', mustBeString);
     let loader = getFlag(stdin, stdinKeys, 'loader', mustBeString);
@@ -363,7 +363,8 @@ function flagsForBuildOptions(
     if (sourcefile) flags.push(`--sourcefile=${sourcefile}`);
     if (loader) flags.push(`--loader=${loader}`);
     if (resolveDir) stdinResolveDir = resolveDir + '';
-    stdinContents = contents ? contents + '' : '';
+    if (typeof contents === 'string') stdinContents = protocol.encodeUTF8(contents)
+    else if (contents instanceof Uint8Array) stdinContents = contents
   }
 
   let nodePaths: string[] = [];
@@ -439,7 +440,7 @@ export interface StreamOut {
 }
 
 export interface StreamFS {
-  writeFile(contents: string, callback: (path: string | null) => void): void;
+  writeFile(contents: string | Uint8Array, callback: (path: string | null) => void): void;
   readFile(path: string, callback: (err: Error | null, contents: string | null) => void): void;
 }
 
@@ -462,7 +463,7 @@ export interface StreamService {
   transform(args: {
     callName: string,
     refs: Refs | null,
-    input: string,
+    input: string | Uint8Array,
     options: types.TransformOptions,
     isTTY: boolean,
     fs: StreamFS,
@@ -1348,7 +1349,8 @@ export function createChannel(streamIn: StreamIn): StreamOut {
     // that doesn't work.
     let start = (inputPath: string | null) => {
       try {
-        if (typeof input !== 'string') throw new Error('The input to "transform" must be a string');
+        if (typeof input !== 'string' && !(input instanceof Uint8Array))
+          throw new Error('The input to "transform" must be a string or a Uint8Array');
         let {
           flags,
           mangleCache,
@@ -1357,7 +1359,9 @@ export function createChannel(streamIn: StreamIn): StreamOut {
           command: 'transform',
           flags,
           inputFS: inputPath !== null,
-          input: inputPath !== null ? inputPath : input,
+          input: inputPath !== null ? protocol.encodeUTF8(inputPath)
+            : typeof input === 'string' ? protocol.encodeUTF8(input)
+              : input,
         };
         if (mangleCache) request.mangleCache = mangleCache;
         sendRequest<protocol.TransformRequest, protocol.TransformResponse>(refs, request, (error, response) => {
@@ -1412,7 +1416,7 @@ export function createChannel(streamIn: StreamIn): StreamOut {
         });
       }
     };
-    if (typeof input === 'string' && input.length > 1024 * 1024) {
+    if ((typeof input === 'string' || input instanceof Uint8Array) && input.length > 1024 * 1024) {
       let next = start;
       start = () => fs.writeFile(input, next);
     }

lib/shared/stdio_protocol.ts

@@ -12,7 +12,7 @@ export interface BuildRequest {
   entries: [string, string][]; // Use an array instead of a map to preserve order
   flags: string[];
   write: boolean;
-  stdinContents: string | null;
+  stdinContents: Uint8Array | null;
   stdinResolveDir: string | null;
   absWorkingDir: string;
   incremental: boolean;
@@ -100,7 +100,7 @@ export interface OnWatchRebuildRequest {
 export interface TransformRequest {
   command: 'transform';
   flags: string[];
-  input: string;
+  input: Uint8Array;
   inputFS: boolean;
   mangleCache?: Record<string, string | false>;
 }

lib/shared/types.ts

@@ -147,7 +147,7 @@ export interface WatchMode {
 }
 
 export interface StdinOptions {
-  contents: string;
+  contents: string | Uint8Array;
   resolveDir?: string;
   sourcefile?: string;
   loader?: Loader;
@@ -499,7 +499,7 @@ export declare function serve(serveOptions: ServeOptions, buildOptions: BuildOpt
  *
  * Documentation: https://esbuild.github.io/api/#transform-api
  */
-export declare function transform(input: string, options?: TransformOptions): Promise<TransformResult>;
+export declare function transform(input: string | Uint8Array, options?: TransformOptions): Promise<TransformResult>;
 
 /**
  * Converts log messages to formatted message strings suitable for printing in

scripts/js-api-tests.js

@@ -599,6 +599,28 @@ let buildTests = {
     assert.strictEqual(result.__esModule, true)
   },
 
+  async buildLoaderStdinBase64({ esbuild }) {
+    // UTF-16
+    var result = await esbuild.build({
+      stdin: {
+        contents: `\xFF`,
+        loader: 'base64',
+      },
+      write: false,
+    })
+    assert.strictEqual(result.outputFiles[0].text, `module.exports = "w78=";\n`)
+
+    // Binary
+    var result = await esbuild.build({
+      stdin: {
+        contents: new Uint8Array([0xFF]),
+        loader: 'base64',
+      },
+      write: false,
+    })
+    assert.strictEqual(result.outputFiles[0].text, `module.exports = "/w==";\n`)
+  },
+
   async fileLoader({ esbuild, testDir }) {
     const input = path.join(testDir, 'in.js')
     const data = path.join(testDir, 'data.bin')
@@ -3147,10 +3169,10 @@ async function futureSyntax(esbuild, js, targetBelow, targetAbove) {
 let transformTests = {
   async transformWithNonString({ esbuild }) {
     try {
-      await esbuild.transform(Buffer.from(`1+2`))
+      await esbuild.transform(Object.create({ toString() { return '1+2' } }))
       throw new Error('Expected an error to be thrown');
     } catch (e) {
-      assert.strictEqual(e.errors[0].text, 'The input to "transform" must be a string')
+      assert.strictEqual(e.errors ? e.errors[0].text : e + '', 'The input to "transform" must be a string or a Uint8Array')
     }
   },
 
@@ -3176,6 +3198,16 @@ let transformTests = {
     }
   },
 
+  async transformLoaderBase64({ esbuild }) {
+    // UTF-16
+    var result = await esbuild.transform(`\xFF`, { loader: 'base64' })
+    assert.strictEqual(result.code, `module.exports = "w78=";\n`)
+
+    // Binary
+    var result = await esbuild.transform(new Uint8Array([0xFF]), { loader: 'base64' })
+    assert.strictEqual(result.code, `module.exports = "/w==";\n`)
+  },
+
   async avoidTDZ({ esbuild }) {
     var { code } = await esbuild.transform(`
       class Foo {
@@ -4841,10 +4873,10 @@ let syncTests = {
 
   async transformSyncWithNonString({ esbuild }) {
     try {
-      esbuild.transformSync(Buffer.from(`1+2`))
+      esbuild.transformSync(Object.create({ toString() { return '1+2' } }))
       throw new Error('Expected an error to be thrown');
     } catch (e) {
-      assert.strictEqual(e.errors[0].text, 'The input to "transform" must be a string')
+      assert.strictEqual(e.errors ? e.errors[0].text : e + '', 'The input to "transform" must be a string or a Uint8Array')
     }
   },