Add support for Node native addons (#837)

README.md

@@ -214,10 +214,17 @@ This way you may even avoid creating `pkg` config for your project.
 
 ## Native addons
 
-Native addons (`.node` files) use is supported, but packaging
-`.node` files inside the executable is not resolved yet. You have
-to deploy native addons used by your project to the same directory
-as the executable.
+Native addons (`.node` files) use is supported. When `pkg` encounters
+a `.node` file in a `require` call, it will package this like an asset.
+In some cases (like with the `bindings` package), the module path is generated
+dynamicaly and `pkg` won't be able to detect it. In this case, you should
+add the `.node` file directly in the `assets` field in `package.json`.
+
+The way Node.js requires native addon is different from a classic JS
+file. It needs to have a file on disk to load it, but `pkg` only generates
+one file. To circumvent this, `pkg` will create a temporary file on the
+disk. These files will stay on the disk after the process has exited
+and will be used again on the next process launch.
 
 When a package, that contains a native module, is being installed,
 the native module is compiled against current system-wide Node.js

lib/packer.js

@@ -2,7 +2,7 @@
 
 import {
   STORE_BLOB, STORE_CONTENT, STORE_LINKS,
-  STORE_STAT, isDotJS, isDotJSON, isDotNODE
+  STORE_STAT, isDotJS, isDotJSON
 } from '../prelude/common.js';
 
 import { log, wasReported } from './log.js';
@@ -34,18 +34,13 @@ function hasAnyStore (record) {
 
 export default function ({ records, entrypoint, bytecode }) {
   const stripes = [];
-
   for (const snap in records) {
     const record = records[snap];
     const { file } = record;
     if (!hasAnyStore(record)) continue;
     assert(record[STORE_STAT], 'packer: no STORE_STAT');
 
-    if (isDotNODE(file)) {
-      continue;
-    } else {
-      assert(record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]);
-    }
+    assert(record[STORE_BLOB] || record[STORE_CONTENT] || record[STORE_LINKS]);
 
     if (record[STORE_BLOB] && !bytecode) {
       delete record[STORE_BLOB];

lib/walker.js

@@ -522,24 +522,11 @@ class Walker {
       store: STORE_STAT
     });
 
-    if (isDotNODE(record.file)) {
-      // provide explicit deployFiles to override
-      // native addon deployment place. see 'sharp'
-      if (!marker.hasDeployFiles) {
-        log.warn('Cannot include addon %1 into executable.', [
-          'The addon must be distributed with executable as %2.',
-          '%1: ' + record.file,
-          '%2: path-to-executable/' + path.basename(record.file) ]);
-      }
-      return; // discard
-    }
-
     const derivatives1 = [];
     await this.stepActivate(marker, derivatives1);
     await this.stepDerivatives(record, marker, derivatives1);
-
     if (store === STORE_BLOB) {
-      if (unlikelyJavascript(record.file)) {
+      if (unlikelyJavascript(record.file) || isDotNODE(record.file)) {
         this.append({
           file: record.file,
           marker,

prelude/bootstrap.js

@@ -1580,3 +1580,65 @@ function payloadFileSync (pointer) {
     });
   }
 }());
+
+// /////////////////////////////////////////////////////////////////
+// PATCH PROCESS ///////////////////////////////////////////////////
+// /////////////////////////////////////////////////////////////////
+
+(function () {
+  const fs = require('fs');
+  var ancestor = {};
+  ancestor.dlopen = process.dlopen;
+
+  process.dlopen = function () {
+    const args = cloneArgs(arguments);
+    const modulePath = args[1];
+    const moduleDirname = require('path').dirname(modulePath);
+    if (insideSnapshot(modulePath)) {
+      // Node addon files and .so cannot be read with fs directly, they are loaded with process.dlopen which needs a filesystem path
+      // we need to write the file somewhere on disk first and then load it
+      const moduleContent = fs.readFileSync(modulePath);
+      const moduleBaseName = require('path').basename(modulePath);
+      const hash = require('crypto').createHash('sha256').update(moduleContent).digest('hex');
+      const tmpModulePath = `${require('os').tmpdir()}/${hash}_${moduleBaseName}`;
+      try {
+        fs.statSync(tmpModulePath);
+      } catch (e) {
+        // Most likely this means the module is not on disk yet
+        fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 });
+      }
+      args[1] = tmpModulePath;
+    }
+
+    const unknownModuleErrorRegex = /([^:]+): cannot open shared object file: No such file or directory/;
+    const tryImporting = function (previousErrorMessage) {
+      try {
+        const res = ancestor.dlopen.apply(process, args);
+        return res;
+      } catch (e) {
+        if (e.message === previousErrorMessage) {
+          // we already tried to fix this and it didn't work, give up
+          throw e;
+        }
+        if (e.message.match(unknownModuleErrorRegex)) {
+          // some modules are packaged with dynamic linking and needs to open other files that should be in
+          // the same directory, in this case, we write this file in the same /tmp directory and try to
+          // import the module again
+          const moduleName = e.message.match(unknownModuleErrorRegex)[1];
+          const importModulePath = `${moduleDirname}/${moduleName}`;
+          const moduleContent = fs.readFileSync(importModulePath);
+          const moduleBaseName = require('path').basename(importModulePath);
+          const tmpModulePath = `${require('os').tmpdir()}/${moduleBaseName}`;
+          try {
+            fs.statSync(tmpModulePath);
+          } catch (err) {
+            fs.writeFileSync(tmpModulePath, moduleContent, { mode: 0o444 });
+          }
+          return tryImporting(e.message);
+        }
+        throw e;
+      }
+    };
+    tryImporting();
+  };
+}());

test/test-50-cannot-include-addon/main.js → test/test-50-can-include-addon/main.js

@@ -26,6 +26,6 @@ right = utils.pkg.sync([
 
 assert(right.indexOf('\x1B\x5B') < 0, 'colors detected');
 right = right.replace(/\\/g, '/');
-assert(right.indexOf('test-50-cannot-include-addon/time.node') >= 0);
-assert(right.indexOf('path-to-executable/time.node') >= 0);
+assert(right.indexOf('test-50-can-include-addon/time.node') === -1);
+assert(right.indexOf('path-to-executable/time.node') === -1);
 utils.vacuum.sync(output);

test/test-50-can-include-addon/time.node

@@ -0,0 +1 @@
+module.exports = 'test';

test/test-50-native-addon-3/lib/community/time-y.node

@@ -1 +1 @@
-module.exports = require('path').basename(__filename);
+module.exports = 'time-y';

test/test-50-native-addon-3/lib/enterprise/time-z.node

@@ -1 +1 @@
-module.exports = require('path').basename(__filename);
+module.exports = 'time-z';

test/test-50-native-addon-3/lib/time-x.node

@@ -1 +1 @@
-module.exports = require('path').basename(__filename);
+module.exports = 'test';

test/test-50-native-addon-4/lib/time.node

@@ -0,0 +1 @@
+module.exports = 'test';

test/test-50-native-addon-4/main.js

@@ -0,0 +1,36 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const path = require('path');
+const assert = require('assert');
+const utils = require('../utils.js');
+
+assert(!module.parent);
+assert(__dirname === process.cwd());
+
+const host = 'node' + process.version.match(/^v(\d+)/)[1];
+const target = process.argv[2] || host;
+const input = './test-x-index.js';
+const output = './run-time/test-output.exe';
+
+let left, right;
+utils.mkdirp.sync(path.dirname(output));
+
+left = utils.spawn.sync(
+  'node', [ path.basename(input) ],
+  { cwd: path.dirname(input) }
+);
+
+utils.pkg.sync([
+  '--target', target,
+  '--output', output, input
+]);
+
+right = utils.spawn.sync(
+  './' + path.basename(output), [],
+  { cwd: path.dirname(output) }
+);
+
+assert.equal(left, right);
+utils.vacuum.sync(path.dirname(output));

test/test-50-native-addon-4/test-x-index.js

@@ -0,0 +1,10 @@
+/* eslint-disable no-underscore-dangle */
+
+'use strict';
+
+var fs = require('fs');
+var path = require('path');
+var Module = require('module');
+Module._extensions['.node'] = Module._extensions['.js'];
+console.log(fs.existsSync(path.join(__dirname, 'lib/time.node')));
+console.log(require('./lib/time.node'));

test/test-50-native-addon/lib/time.node

@@ -1 +1 @@
-module.exports = require('path').basename(__filename);
+module.exports = 'test';