From 44ecd418923310aafbef8c6ccf9a9920bcf7af85 Mon Sep 17 00:00:00 2001
From: Joyee Cheung <joyeec9h3@gmail.com>
Date: Fri, 18 Jun 2021 05:01:41 +0800
Subject: [PATCH] build: pass directory instead of list of files to js2c.py

On Windows there is a limit to the length of commands, so there
will be an error once the lengths of the JS file names combined
exceed that limit. This patch modifies js2c.py so that
it now takes a --directory argument to glob for .js and
.mjs files in addition to the list of files passed directly.
We still pass the additional files we include from deps/
directly through the command line, as we only includes some of
them so we cannot simply glob, but those are limited so listing
them out should be fine.

Refs: https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation

PR-URL: https://github.com/nodejs/node/pull/39069
Refs: https://github.com/nodejs/node/pull/38971
Reviewed-By: Chengzhong Wu <legendecas@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
---
 node.gyp              | 265 +++---------------------------------------
 tools/js2c.py         |  14 ++-
 tools/search_files.py |  22 ++++
 tools/utils.py        |   9 ++
 4 files changed, 63 insertions(+), 247 deletions(-)
 create mode 100644 tools/search_files.py

diff --git a/node.gyp b/node.gyp
index 1de9edc07f610d..bb80fa30bc8fe1 100644
--- a/node.gyp
+++ b/node.gyp
@@ -27,251 +27,15 @@
     'node_lib_target_name%': 'libnode',
     'node_intermediate_lib_type%': 'static_library',
     'node_builtin_modules_path%': '',
+    # We list the deps/ files out instead of globbing them in js2c.py since we
+    # only include a subset of all the files under these directories.
+    # The lengths of their file names combined should not exceed the
+    # Windows command length limit or there would be an error.
+    # See https://docs.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/command-line-string-limitation
     'library_files': [
-      'lib/internal/bootstrap/environment.js',
-      'lib/internal/bootstrap/loaders.js',
-      'lib/internal/bootstrap/node.js',
-      'lib/internal/bootstrap/pre_execution.js',
-      'lib/internal/bootstrap/switches/does_own_process_state.js',
-      'lib/internal/bootstrap/switches/does_not_own_process_state.js',
-      'lib/internal/bootstrap/switches/is_main_thread.js',
-      'lib/internal/bootstrap/switches/is_not_main_thread.js',
-      'lib/internal/per_context/primordials.js',
-      'lib/internal/per_context/domexception.js',
-      'lib/internal/per_context/messageport.js',
-      'lib/async_hooks.js',
-      'lib/assert.js',
-      'lib/assert/strict.js',
-      'lib/buffer.js',
-      'lib/child_process.js',
-      'lib/console.js',
-      'lib/constants.js',
-      'lib/crypto.js',
-      'lib/cluster.js',
-      'lib/diagnostics_channel.js',
-      'lib/dgram.js',
-      'lib/dns.js',
-      'lib/dns/promises.js',
-      'lib/domain.js',
-      'lib/events.js',
-      'lib/fs.js',
-      'lib/fs/promises.js',
-      'lib/http.js',
-      'lib/http2.js',
-      'lib/_http_agent.js',
-      'lib/_http_client.js',
-      'lib/_http_common.js',
-      'lib/_http_incoming.js',
-      'lib/_http_outgoing.js',
-      'lib/_http_server.js',
-      'lib/https.js',
-      'lib/inspector.js',
-      'lib/module.js',
-      'lib/net.js',
-      'lib/os.js',
-      'lib/path.js',
-      'lib/path/posix.js',
-      'lib/path/win32.js',
-      'lib/perf_hooks.js',
-      'lib/process.js',
-      'lib/punycode.js',
-      'lib/querystring.js',
-      'lib/readline.js',
-      'lib/repl.js',
-      'lib/stream.js',
-      'lib/stream/promises.js',
-      'lib/_stream_readable.js',
-      'lib/_stream_writable.js',
-      'lib/_stream_duplex.js',
-      'lib/_stream_transform.js',
-      'lib/_stream_passthrough.js',
-      'lib/_stream_wrap.js',
-      'lib/string_decoder.js',
-      'lib/sys.js',
-      'lib/timers/promises.js',
-      'lib/timers.js',
-      'lib/tls.js',
-      'lib/_tls_common.js',
-      'lib/_tls_wrap.js',
-      'lib/trace_events.js',
-      'lib/tty.js',
-      'lib/url.js',
-      'lib/util.js',
-      'lib/util/types.js',
-      'lib/v8.js',
-      'lib/vm.js',
-      'lib/wasi.js',
-      'lib/worker_threads.js',
-      'lib/zlib.js',
-      'lib/internal/abort_controller.js',
-      'lib/internal/assert.js',
-      'lib/internal/assert/assertion_error.js',
-      'lib/internal/assert/calltracker.js',
-      'lib/internal/async_hooks.js',
-      'lib/internal/blob.js',
-      'lib/internal/blocklist.js',
-      'lib/internal/buffer.js',
-      'lib/internal/cli_table.js',
-      'lib/internal/child_process.js',
-      'lib/internal/child_process/serialization.js',
-      'lib/internal/cluster/child.js',
-      'lib/internal/cluster/primary.js',
-      'lib/internal/cluster/round_robin_handle.js',
-      'lib/internal/cluster/shared_handle.js',
-      'lib/internal/cluster/utils.js',
-      'lib/internal/cluster/worker.js',
-      'lib/internal/console/constructor.js',
-      'lib/internal/console/global.js',
-      'lib/internal/crypto/aes.js',
-      'lib/internal/crypto/certificate.js',
-      'lib/internal/crypto/cipher.js',
-      'lib/internal/crypto/diffiehellman.js',
-      'lib/internal/crypto/dsa.js',
-      'lib/internal/crypto/ec.js',
-      'lib/internal/crypto/hash.js',
-      'lib/internal/crypto/hashnames.js',
-      'lib/internal/crypto/hkdf.js',
-      'lib/internal/crypto/keygen.js',
-      'lib/internal/crypto/keys.js',
-      'lib/internal/crypto/mac.js',
-      'lib/internal/crypto/pbkdf2.js',
-      'lib/internal/crypto/random.js',
-      'lib/internal/crypto/rsa.js',
-      'lib/internal/crypto/scrypt.js',
-      'lib/internal/crypto/sig.js',
-      'lib/internal/crypto/util.js',
-      'lib/internal/crypto/webcrypto.js',
-      'lib/internal/crypto/x509.js',
-      'lib/internal/constants.js',
-      'lib/internal/debugger/_inspect.js',
-      'lib/internal/debugger/inspect_client.js',
-      'lib/internal/debugger/inspect_repl.js',
-      'lib/internal/dgram.js',
-      'lib/internal/dns/promises.js',
-      'lib/internal/dns/utils.js',
-      'lib/internal/dtrace.js',
-      'lib/internal/encoding.js',
-      'lib/internal/errors.js',
-      'lib/internal/error_serdes.js',
-      'lib/internal/event_target.js',
-      'lib/internal/fixed_queue.js',
-      'lib/internal/freelist.js',
-      'lib/internal/freeze_intrinsics.js',
-      'lib/internal/fs/dir.js',
-      'lib/internal/fs/promises.js',
-      'lib/internal/fs/read_file_context.js',
-      'lib/internal/fs/rimraf.js',
-      'lib/internal/fs/streams.js',
-      'lib/internal/fs/sync_write_stream.js',
-      'lib/internal/fs/utils.js',
-      'lib/internal/fs/watchers.js',
-      'lib/internal/http.js',
-      'lib/internal/heap_utils.js',
-      'lib/internal/histogram.js',
-      'lib/internal/idna.js',
-      'lib/internal/inspector_async_hook.js',
-      'lib/internal/js_stream_socket.js',
-      'lib/internal/legacy/processbinding.js',
-      'lib/internal/linkedlist.js',
-      'lib/internal/main/check_syntax.js',
-      'lib/internal/main/eval_string.js',
-      'lib/internal/main/eval_stdin.js',
-      'lib/internal/main/inspect.js',
-      'lib/internal/main/print_help.js',
-      'lib/internal/main/prof_process.js',
-      'lib/internal/main/repl.js',
-      'lib/internal/main/run_main_module.js',
-      'lib/internal/main/worker_thread.js',
-      'lib/internal/modules/run_main.js',
-      'lib/internal/modules/package_json_reader.js',
-      'lib/internal/modules/cjs/helpers.js',
-      'lib/internal/modules/cjs/loader.js',
-      'lib/internal/modules/esm/loader.js',
-      'lib/internal/modules/esm/create_dynamic_module.js',
-      'lib/internal/modules/esm/get_format.js',
-      'lib/internal/modules/esm/get_source.js',
-      'lib/internal/modules/esm/module_job.js',
-      'lib/internal/modules/esm/module_map.js',
-      'lib/internal/modules/esm/resolve.js',
-      'lib/internal/modules/esm/transform_source.js',
-      'lib/internal/modules/esm/translators.js',
-      'lib/internal/net.js',
-      'lib/internal/options.js',
-      'lib/internal/perf/perf.js',
-      'lib/internal/perf/nodetiming.js',
-      'lib/internal/perf/usertiming.js',
-      'lib/internal/perf/observe.js',
-      'lib/internal/perf/event_loop_delay.js',
-      'lib/internal/perf/event_loop_utilization.js',
-      'lib/internal/perf/timerify.js',
-      'lib/internal/policy/manifest.js',
-      'lib/internal/policy/sri.js',
-      'lib/internal/priority_queue.js',
-      'lib/internal/process/esm_loader.js',
-      'lib/internal/process/execution.js',
-      'lib/internal/process/per_thread.js',
-      'lib/internal/process/policy.js',
-      'lib/internal/process/promises.js',
-      'lib/internal/process/warning.js',
-      'lib/internal/process/worker_thread_only.js',
-      'lib/internal/process/report.js',
-      'lib/internal/process/signal.js',
-      'lib/internal/process/task_queues.js',
-      'lib/internal/querystring.js',
-      'lib/internal/readline/callbacks.js',
-      'lib/internal/readline/emitKeypressEvents.js',
-      'lib/internal/readline/utils.js',
-      'lib/internal/repl.js',
-      'lib/internal/repl/await.js',
-      'lib/internal/repl/history.js',
-      'lib/internal/repl/utils.js',
-      'lib/internal/socketaddress.js',
-      'lib/internal/socket_list.js',
-      'lib/internal/source_map/prepare_stack_trace.js',
-      'lib/internal/source_map/source_map.js',
-      'lib/internal/source_map/source_map_cache.js',
-      'lib/internal/test/binding.js',
-      'lib/internal/test/transfer.js',
-      'lib/internal/timers.js',
-      'lib/internal/tls.js',
-      'lib/internal/trace_events_async_hooks.js',
-      'lib/internal/tty.js',
-      'lib/internal/url.js',
-      'lib/internal/util.js',
-      'lib/internal/util/comparisons.js',
-      'lib/internal/util/debuglog.js',
-      'lib/internal/util/inspect.js',
-      'lib/internal/util/inspector.js',
-      'lib/internal/util/iterable_weak_map.js',
-      'lib/internal/util/types.js',
-      'lib/internal/http2/core.js',
-      'lib/internal/http2/compat.js',
-      'lib/internal/http2/util.js',
-      'lib/internal/v8_prof_polyfill.js',
-      'lib/internal/v8_prof_processor.js',
-      'lib/internal/validators.js',
-      'lib/internal/stream_base_commons.js',
-      'lib/internal/vm/module.js',
-      'lib/internal/worker.js',
-      'lib/internal/worker/io.js',
-      'lib/internal/worker/js_transferable.js',
-      'lib/internal/watchdog.js',
-      'lib/internal/streams/lazy_transform.js',
-      'lib/internal/streams/add-abort-signal.js',
-      'lib/internal/streams/buffer_list.js',
-      'lib/internal/streams/duplexpair.js',
-      'lib/internal/streams/from.js',
-      'lib/internal/streams/legacy.js',
-      'lib/internal/streams/readable.js',
-      'lib/internal/streams/writable.js',
-      'lib/internal/streams/duplex.js',
-      'lib/internal/streams/passthrough.js',
-      'lib/internal/streams/transform.js',
-      'lib/internal/streams/destroy.js',
-      'lib/internal/streams/state.js',
-      'lib/internal/streams/pipeline.js',
-      'lib/internal/streams/end-of-stream.js',
-      'lib/internal/streams/utils.js',
+      '<!@(python tools/search_files.py --ext js lib)',
+    ],
+    'deps_files': [
       'deps/v8/tools/splaytree.mjs',
       'deps/v8/tools/codemap.mjs',
       'deps/v8/tools/consarray.mjs',
@@ -794,6 +558,7 @@
         'deps/v8/include/v8.h',
         # javascript files to make for an even more pleasant IDE experience
         '<@(library_files)',
+        '<@(deps_files)',
         # node.gyp is added by default, common.gypi is added for change detection
         'common.gypi',
       ],
@@ -1047,14 +812,21 @@
             # Put the code first so it's a dependency and can be used for invocation.
             'tools/js2c.py',
             '<@(library_files)',
+            '<@(deps_files)',
             'config.gypi'
           ],
           'outputs': [
             '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
           ],
           'action': [
-            'python', '<@(_inputs)',
-            '--target', '<@(_outputs)',
+            'python',
+            'tools/js2c.py',
+            '--directory',
+            'lib',
+            '--target',
+            '<@(_outputs)',
+            'config.gypi',
+            '<@(deps_files)',
           ],
         },
       ],
@@ -1598,6 +1370,7 @@
           ],
           'sources': [
             '<@(library_files)',
+            '<@(deps_files)',
             'common.gypi',
           ],
           'direct_dependent_settings': {
diff --git a/tools/js2c.py b/tools/js2c.py
index d40f28ce2bff2b..d93be2123e0f8c 100755
--- a/tools/js2c.py
+++ b/tools/js2c.py
@@ -36,6 +36,7 @@
 import re
 import functools
 import codecs
+import utils
 
 def ReadFile(filename):
   if is_verbose:
@@ -204,12 +205,23 @@ def main():
     fromfile_prefix_chars='@'
   )
   parser.add_argument('--target', help='output file')
+  parser.add_argument(
+      '--directory',
+      default=None,
+      help='input file directory')
   parser.add_argument('--verbose', action='store_true', help='output file')
   parser.add_argument('sources', nargs='*', help='input files')
   options = parser.parse_args()
   global is_verbose
   is_verbose = options.verbose
-  source_files = functools.reduce(SourceFileByExt, options.sources, {})
+  sources = options.sources
+  if options.directory is not None:
+    js_files = utils.SearchFiles(options.directory, 'js')
+    mjs_files = utils.SearchFiles(options.directory, 'mjs')
+    sources = js_files + mjs_files + options.sources
+
+  source_files = functools.reduce(SourceFileByExt, sources, {})
+
   # Should have exactly 3 types: `.js`, `.mjs` and `.gypi`
   assert len(source_files) == 3
   # Currently config.gypi is the only `.gypi` file allowed
diff --git a/tools/search_files.py b/tools/search_files.py
new file mode 100644
index 00000000000000..e236eee47dc852
--- /dev/null
+++ b/tools/search_files.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+
+"""
+This is a utility for recursively searching files under
+a specified directory
+"""
+
+import argparse
+import utils
+
+def main():
+  parser = argparse.ArgumentParser(
+    description='Search files with a specific extension under a directory',
+    fromfile_prefix_chars='@'
+  )
+  parser.add_argument('--ext', required=True, help='extension to search for')
+  parser.add_argument('directory', help='input directory')
+  options = parser.parse_args()
+  print('\n'.join(utils.SearchFiles(options.directory, options.ext)))
+
+if __name__ == "__main__":
+  main()
diff --git a/tools/utils.py b/tools/utils.py
index 9734836db7f06e..e9b38c87b04b70 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -26,8 +26,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+import glob
 import platform
 import re
+import sys
 
 
 # Reads a .list file into an array of strings
@@ -106,3 +108,10 @@ def GuessArchitecture():
 
 def IsWindows():
   return GuessOS() == 'win32'
+
+
+def SearchFiles(dir, ext):
+  list = glob.glob(dir+ '/**/*.' + ext, recursive=True)
+  if sys.platform == 'win32':
+    list = [ x.replace('\\', '/')for x in list]
+  return list