From 3baa4e4172c47052ee7e942cef542c1a7dea75aa Mon Sep 17 00:00:00 2001 From: Samuel Attard Date: Tue, 14 Jul 2020 19:41:49 -0700 Subject: [PATCH] gyp: update gyp to 0.4.0 Reviewed-By: Rod Vagg Reviewed-By: Jiawen Geng PR-URL: https://github.com/nodejs/node-gyp/pull/2165 --- gyp/.github/workflows/Python_tests.yml | 4 +- gyp/CHANGELOG.md | 41 +++++++++++ gyp/pylib/gyp/generator/android.py | 4 +- gyp/pylib/gyp/generator/make.py | 14 +++- gyp/pylib/gyp/generator/msvs.py | 96 ++++++++++++++++++-------- gyp/pylib/gyp/generator/ninja.py | 14 ++-- gyp/pylib/gyp/input.py | 18 ++--- gyp/pylib/gyp/xcode_emulation.py | 7 ++ gyp/setup.py | 28 ++++---- gyp/test_gyp.py | 3 +- 10 files changed, 162 insertions(+), 67 deletions(-) create mode 100644 gyp/CHANGELOG.md diff --git a/gyp/.github/workflows/Python_tests.yml b/gyp/.github/workflows/Python_tests.yml index 47c40343ad..a93b92f426 100644 --- a/gyp/.github/workflows/Python_tests.yml +++ b/gyp/.github/workflows/Python_tests.yml @@ -14,9 +14,9 @@ jobs: os: [macos-latest, ubuntu-latest] # , windows-latest] python-version: [2.7, 3.6, 3.7, 3.8] # 3.5, steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/gyp/CHANGELOG.md b/gyp/CHANGELOG.md new file mode 100644 index 0000000000..8cbcdd3b72 --- /dev/null +++ b/gyp/CHANGELOG.md @@ -0,0 +1,41 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). + +## [Unreleased] + +## [0.4.0] - 2020-07-14 + +### Added +- Added support for passing arbitrary architectures to Xcode builds, enables `arm64` builds. + +### Fixed +- Fixed a bug on Solaris where copying archives failed. + +## [0.3.0] - 2020-06-06 + +### Added +- Added support for MSVC cross-compilation. This allows compilation on x64 for + a Windows ARM target. + +### Fixed +- Fixed XCode CLT version detection on macOS Catalina. + +## [0.2.1] - 2020-05-05 + +### Fixed +- Relicensed to Node.js contributors. +- Fixed Windows bug introduced in v0.2.0. + +## [0.2.0] - 2020-04-06 + +This is the first release of this project, based on https://chromium.googlesource.com/external/gyp +with changes made over the years in Node.js and node-gyp. + +[Unreleased]: https://github.com/nodejs/gyp-next/compare/v0.4.0...HEAD +[0.4.0]: https://github.com/nodejs/gyp-next/compare/v0.3.0...v0.4.0 +[0.3.0]: https://github.com/nodejs/gyp-next/compare/v0.2.1...v0.3.0 +[0.2.1]: https://github.com/nodejs/gyp-next/compare/v0.2.0...v0.2.1 +[0.2.0]: https://github.com/nodejs/gyp-next/releases/tag/v0.2.0 diff --git a/gyp/pylib/gyp/generator/android.py b/gyp/pylib/gyp/generator/android.py index 7dbbd579ca..3ac61008b9 100644 --- a/gyp/pylib/gyp/generator/android.py +++ b/gyp/pylib/gyp/generator/android.py @@ -981,9 +981,9 @@ def WriteList( """ values = "" if value_list: - value_list = [quoter(prefix + l) for l in value_list] + value_list = [quoter(prefix + value) for value in value_list] if local_pathify: - value_list = [self.LocalPathify(l) for l in value_list] + value_list = [self.LocalPathify(value) for value in value_list] values = " \\\n\t" + " \\\n\t".join(value_list) self.fp.write("%s :=%s\n\n" % (variable, values)) diff --git a/gyp/pylib/gyp/generator/make.py b/gyp/pylib/gyp/generator/make.py index 4a50b04339..6e1c5205cf 100644 --- a/gyp/pylib/gyp/generator/make.py +++ b/gyp/pylib/gyp/generator/make.py @@ -1342,7 +1342,7 @@ def WriteSources( ) if self.flavor == "mac": - cflags = self.xcode_settings.GetCflags(configname) + cflags = self.xcode_settings.GetCflags(configname, arch=config.get('xcode_configuration_platform')) cflags_c = self.xcode_settings.GetCflagsC(configname) cflags_cc = self.xcode_settings.GetCflagsCC(configname) cflags_objc = self.xcode_settings.GetCflagsObjC(configname) @@ -1637,6 +1637,7 @@ def WriteTarget( configname, generator_default_variables["PRODUCT_DIR"], lambda p: Sourceify(self.Absolutify(p)), + arch=config.get('xcode_configuration_platform') ) # TARGET_POSTBUILDS_$(BUILDTYPE) is added to postbuilds later on. @@ -1944,7 +1945,7 @@ def WriteList(self, value_list, variable=None, prefix="", quoter=QuoteIfNecessar """ values = "" if value_list: - value_list = [quoter(prefix + l) for l in value_list] + value_list = [quoter(prefix + value) for value in value_list] values = " \\\n\t" + " \\\n\t".join(value_list) self.fp.write("%s :=%s\n\n" % (variable, values)) @@ -2362,7 +2363,14 @@ def CalculateMakefilePath(build_file, base_name): } ) elif flavor == "solaris": - header_params.update({"flock": "./gyp-flock-tool flock", "flock_index": 2}) + copy_archive_arguments = "-pPRf@" + header_params.update( + { + "copy_archive_args": copy_archive_arguments, + "flock": "./gyp-flock-tool flock", + "flock_index": 2 + } + ) elif flavor == "freebsd": # Note: OpenBSD has sysutils/flock. lockf seems to be FreeBSD specific. header_params.update({"flock": "lockf"}) diff --git a/gyp/pylib/gyp/generator/msvs.py b/gyp/pylib/gyp/generator/msvs.py index 3a8aac0eb5..5bfceb05cb 100644 --- a/gyp/pylib/gyp/generator/msvs.py +++ b/gyp/pylib/gyp/generator/msvs.py @@ -39,6 +39,7 @@ # letters. VALID_MSVS_GUID_CHARS = re.compile(r"^[A-F0-9\-]+$") +generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested() generator_default_variables = { "DRIVER_PREFIX": "", @@ -50,7 +51,7 @@ "STATIC_LIB_SUFFIX": ".lib", "SHARED_LIB_SUFFIX": ".dll", "INTERMEDIATE_DIR": "$(IntDir)", - "SHARED_INTERMEDIATE_DIR": "$(OutDir)obj/global_intermediate", + "SHARED_INTERMEDIATE_DIR": "$(OutDir)/obj/global_intermediate", "OS": "win", "PRODUCT_DIR": "$(OutDir)", "LIB_DIR": "$(OutDir)lib", @@ -1005,7 +1006,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version): return toolset -def _GenerateProject(project, options, version, generator_flags): +def _GenerateProject(project, options, version, generator_flags, spec): """Generates a vcproj file. Arguments: @@ -1023,7 +1024,7 @@ def _GenerateProject(project, options, version, generator_flags): return [] if version.UsesVcxproj(): - return _GenerateMSBuildProject(project, options, version, generator_flags) + return _GenerateMSBuildProject(project, options, version, generator_flags, spec) else: return _GenerateMSVSProject(project, options, version, generator_flags) @@ -1903,6 +1904,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat): # Convert into a tree of dicts on path. for p in sln_projects: gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2] + if p.endswith("#host"): + target += "_host" gyp_dir = os.path.dirname(gyp_file) path_dict = _GetPathDict(root, gyp_dir) path_dict[target + ".vcproj"] = project_objects[p] @@ -1921,9 +1924,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version): default_config = _GetDefaultConfiguration(spec) proj_filename = default_config.get("msvs_existing_vcproj") if not proj_filename: - proj_filename = ( - spec["target_name"] + options.suffix + msvs_version.ProjectExtension() - ) + proj_filename = spec["target_name"] + if spec["toolset"] == "host": + proj_filename += "_host" + proj_filename = proj_filename + options.suffix + msvs_version.ProjectExtension() build_file = gyp.common.BuildFile(qualified_target) proj_path = os.path.join(os.path.dirname(build_file), proj_filename) @@ -1948,6 +1952,8 @@ def _GetPlatformOverridesOfProject(spec): _ConfigBaseName(config_name, _ConfigPlatform(c)), platform, ) + if spec["toolset"] == "host" and generator_supports_multiple_toolsets: + fixed_config_fullname = "%s|x64" % (config_name,) config_platform_overrides[config_fullname] = fixed_config_fullname return config_platform_overrides @@ -1968,11 +1974,6 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version): projects = {} for qualified_target in target_list: spec = target_dicts[qualified_target] - if spec["toolset"] != "target": - raise GypError( - "Multiple toolsets not supported in msvs build (target %s)" - % qualified_target - ) proj_path, fixpath_prefix = _GetPathOfProject( qualified_target, spec, options, msvs_version ) @@ -1980,9 +1981,12 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version): overrides = _GetPlatformOverridesOfProject(spec) build_file = gyp.common.BuildFile(qualified_target) # Create object for this project. + target_name = spec["target_name"] + if spec["toolset"] == "host": + target_name += "_host" obj = MSVSNew.MSVSProject( proj_path, - name=spec["target_name"], + name=target_name, guid=guid, spec=spec, build_file=build_file, @@ -2161,7 +2165,10 @@ def GenerateOutput(target_list, target_dicts, data, params): for qualified_target in target_list: spec = target_dicts[qualified_target] for config_name, config in spec["configurations"].items(): - configs.add(_ConfigFullName(config_name, config)) + config_name = _ConfigFullName(config_name, config) + configs.add(config_name) + if config_name == "Release|arm64": + configs.add("Release|x64") configs = list(configs) # Figure out all the projects that will be generated and their guids @@ -2174,12 +2181,15 @@ def GenerateOutput(target_list, target_dicts, data, params): for project in project_objects.values(): fixpath_prefix = project.fixpath_prefix missing_sources.extend( - _GenerateProject(project, options, msvs_version, generator_flags) + _GenerateProject(project, options, msvs_version, generator_flags, spec) ) fixpath_prefix = None for build_file in data: # Validate build_file extension + target_only_configs = configs + if generator_supports_multiple_toolsets: + target_only_configs = [i for i in configs if i.endswith("arm64")] if not build_file.endswith(".gyp"): continue sln_path = os.path.splitext(build_file)[0] + options.suffix + ".sln" @@ -2196,7 +2206,7 @@ def GenerateOutput(target_list, target_dicts, data, params): sln = MSVSNew.MSVSSolution( sln_path, entries=root_entries, - variants=configs, + variants=target_only_configs, websiteProperties=False, version=msvs_version, ) @@ -2930,22 +2940,24 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules): easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True) -def _GetConfigurationAndPlatform(name, settings): +def _GetConfigurationAndPlatform(name, settings, spec): configuration = name.rsplit("_", 1)[0] platform = settings.get("msvs_configuration_platform", "Win32") + if spec["toolset"] == "host" and platform == "arm64": + platform = "x64" # Host-only tools are always built for x64 return (configuration, platform) -def _GetConfigurationCondition(name, settings): +def _GetConfigurationCondition(name, settings, spec): return r"'$(Configuration)|$(Platform)'=='%s|%s'" % _GetConfigurationAndPlatform( - name, settings + name, settings, spec ) -def _GetMSBuildProjectConfigurations(configurations): +def _GetMSBuildProjectConfigurations(configurations, spec): group = ["ItemGroup", {"Label": "ProjectConfigurations"}] for (name, settings) in sorted(configurations.items()): - configuration, platform = _GetConfigurationAndPlatform(name, settings) + configuration, platform = _GetConfigurationAndPlatform(name, settings, spec) designation = "%s|%s" % (configuration, platform) group.append( [ @@ -3033,7 +3045,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file): properties = {} for name, settings in spec["configurations"].items(): msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file) - condition = _GetConfigurationCondition(name, settings) + condition = _GetConfigurationCondition(name, settings, spec) character_set = msbuild_attributes.get("CharacterSet") config_type = msbuild_attributes.get("ConfigurationType") _AddConditionalProperty(properties, condition, "ConfigurationType", config_type) @@ -3064,12 +3076,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset): return properties -def _GetMSBuildPropertySheets(configurations): +def _GetMSBuildPropertySheets(configurations, spec): user_props = r"$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" additional_props = {} props_specified = False for name, settings in sorted(configurations.items()): - configuration = _GetConfigurationCondition(name, settings) + configuration = _GetConfigurationCondition(name, settings, spec) if "msbuild_props" in settings: additional_props[configuration] = _FixPaths(settings["msbuild_props"]) props_specified = True @@ -3222,7 +3234,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file): properties = {} for (name, configuration) in sorted(configurations.items()): - condition = _GetConfigurationCondition(name, configuration) + condition = _GetConfigurationCondition(name, configuration, spec) attributes = _GetMSBuildAttributes(spec, configuration, build_file) msbuild_settings = configuration["finalized_msbuild_settings"] _AddConditionalProperty( @@ -3345,7 +3357,7 @@ def _GetMSBuildToolSettingsSections(spec, configurations): msbuild_settings = configuration["finalized_msbuild_settings"] group = [ "ItemDefinitionGroup", - {"Condition": _GetConfigurationCondition(name, configuration)}, + {"Condition": _GetConfigurationCondition(name, configuration, spec)}, ] for tool_name, tool_settings in sorted(msbuild_settings.items()): # Skip the tool named '' which is a holder of global settings handled @@ -3625,7 +3637,7 @@ def _AddSources2( if precompiled_source == source: condition = _GetConfigurationCondition( - config_name, configuration + config_name, configuration, spec ) detail.append( ["PrecompiledHeader", {"Condition": condition}, "Create"] @@ -3652,7 +3664,21 @@ def _GetMSBuildProjectReferences(project): references = [] if project.dependencies: group = ["ItemGroup"] + added_dependency_set = set() for dependency in project.dependencies: + dependency_spec = dependency.spec + should_skip_dep = False + if project.spec["toolset"] == "target": + if dependency_spec["toolset"] == "host": + if dependency_spec["type"] == "static_library": + should_skip_dep = True + if dependency.name.startswith("run_"): + should_skip_dep = False + if should_skip_dep: + continue + + canonical_name = dependency.name.replace("_host", "") + added_dependency_set.add(canonical_name) guid = dependency.guid project_dir = os.path.split(project.path)[0] relative_path = gyp.common.RelativePath(dependency.path, project_dir) @@ -3675,7 +3701,7 @@ def _GetMSBuildProjectReferences(project): return references -def _GenerateMSBuildProject(project, options, version, generator_flags): +def _GenerateMSBuildProject(project, options, version, generator_flags, spec): spec = project.spec configurations = spec["configurations"] project_dir, project_file_name = os.path.split(project.path) @@ -3774,7 +3800,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): }, ] - content += _GetMSBuildProjectConfigurations(configurations) + content += _GetMSBuildProjectConfigurations(configurations, spec) content += _GetMSBuildGlobalProperties( spec, version, project.guid, project_file_name ) @@ -3789,7 +3815,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags): if spec.get("msvs_enable_marmasm"): content += import_marmasm_props_section content += _GetMSBuildExtensions(props_files_of_rules) - content += _GetMSBuildPropertySheets(configurations) + content += _GetMSBuildPropertySheets(configurations, spec) content += macro_section content += _GetMSBuildConfigurationGlobalProperties( spec, configurations, project.build_file @@ -3893,15 +3919,27 @@ def _GenerateActionsForMSBuild(spec, actions_to_add): sources_handled_by_action = OrderedSet() actions_spec = [] for primary_input, actions in actions_to_add.items(): + if generator_supports_multiple_toolsets: + primary_input = primary_input.replace(".exe", "_host.exe") inputs = OrderedSet() outputs = OrderedSet() descriptions = [] commands = [] for action in actions: + + def fixup_host_exe(i): + if "$(OutDir)" in i: + i = i.replace(".exe", "_host.exe") + return i + + if generator_supports_multiple_toolsets: + action["inputs"] = [fixup_host_exe(i) for i in action["inputs"]] inputs.update(OrderedSet(action["inputs"])) outputs.update(OrderedSet(action["outputs"])) descriptions.append(action["description"]) cmd = action["command"] + if generator_supports_multiple_toolsets: + cmd = cmd.replace(".exe", "_host.exe") # For most actions, add 'call' so that actions that invoke batch files # return and continue executing. msbuild_use_call provides a way to # disable this but I have not seen any adverse effect from doing that diff --git a/gyp/pylib/gyp/generator/ninja.py b/gyp/pylib/gyp/generator/ninja.py index 19e00319a7..384b252e73 100644 --- a/gyp/pylib/gyp/generator/ninja.py +++ b/gyp/pylib/gyp/generator/ninja.py @@ -78,7 +78,7 @@ def QuoteShellArgument(arg, flavor): """Quote a string such that it will be interpreted as a single argument by the shell.""" # Rather than attempting to enumerate the bad shell characters, just - # whitelist common OK ones and quote anything else. + # allow common OK ones and quote anything else. if re.match(r"^[a-zA-Z0-9_=.\\/-]+$", arg): return arg # No quoting necessary. if flavor == "win": @@ -1481,16 +1481,18 @@ def WriteLinkForArch( library_dirs = config.get("library_dirs", []) if self.flavor == "win": library_dirs = [ - self.msvs_settings.ConvertVSMacros(l, config_name) for l in library_dirs + self.msvs_settings.ConvertVSMacros(library_dir, config_name) + for library_dir in library_dirs ] library_dirs = [ - "/LIBPATH:" + QuoteShellArgument(self.GypPathToNinja(l), self.flavor) - for l in library_dirs + "/LIBPATH:" + + QuoteShellArgument(self.GypPathToNinja(library_dir), self.flavor) + for library_dir in library_dirs ] else: library_dirs = [ - QuoteShellArgument("-L" + self.GypPathToNinja(l), self.flavor) - for l in library_dirs + QuoteShellArgument("-L" + self.GypPathToNinja(library_dir), self.flavor) + for library_dir in library_dirs ] libraries = gyp.common.uniquer( diff --git a/gyp/pylib/gyp/input.py b/gyp/pylib/gyp/input.py index 139df75405..00c4ee1f96 100644 --- a/gyp/pylib/gyp/input.py +++ b/gyp/pylib/gyp/input.py @@ -1619,10 +1619,10 @@ def ExpandWildcardDependencies(targets, data): index = index + 1 -def Unify(l): - """Removes duplicate elements from l, keeping the first element.""" +def Unify(items): + """Removes duplicate elements from items, keeping the first element.""" seen = {} - return [seen.setdefault(e, e) for e in l if e not in seen] + return [seen.setdefault(e, e) for e in items if e not in seen] def RemoveDuplicateDependencies(targets): @@ -1635,10 +1635,10 @@ def RemoveDuplicateDependencies(targets): target_dict[dependency_key] = Unify(dependencies) -def Filter(l, item): - """Removes item from l.""" +def Filter(items, item): + """Removes item from items.""" res = {} - return [res.setdefault(e, e) for e in l if e != item] + return [res.setdefault(e, e) for e in items if e != item] def RemoveSelfDependencies(targets): @@ -2242,11 +2242,11 @@ def MergeLists(to, fro, to_file, fro_file, is_paths=False, append=True): def is_hashable(val): return val.__hash__ - # If x is hashable, returns whether x is in s. Else returns whether x is in l. - def is_in_set_or_list(x, s, l): + # If x is hashable, returns whether x is in s. Else returns whether x is in items. + def is_in_set_or_list(x, s, items): if is_hashable(x): return x in s - return x in l + return x in items prepend_index = 0 diff --git a/gyp/pylib/gyp/xcode_emulation.py b/gyp/pylib/gyp/xcode_emulation.py index 9a717d9095..42a4ce47ed 100644 --- a/gyp/pylib/gyp/xcode_emulation.py +++ b/gyp/pylib/gyp/xcode_emulation.py @@ -1541,6 +1541,13 @@ def CLTVersion(): except GypError: continue + regex = re.compile(r'Command Line Tools for Xcode\s+(?P\S+)') + try: + output = GetStdout(["/usr/sbin/softwareupdate", "--history"]) + return re.search(regex, output).groupdict()["version"] + except GypError: + return None + def GetStdoutQuiet(cmdlist): """Returns the content of standard output returned by invoking |cmdlist|. diff --git a/gyp/setup.py b/gyp/setup.py index 0781c59184..0f75a99b18 100755 --- a/gyp/setup.py +++ b/gyp/setup.py @@ -15,7 +15,7 @@ setup( name="gyp-next", - version="0.2.1", + version="0.4.0", description="A fork of the GYP build system for use in the Node.js projects", long_description=long_description, long_description_content_type="text/markdown", @@ -27,18 +27,18 @@ entry_points={"console_scripts": ["gyp=gyp:script_main"]}, python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*", classifiers=[ - 'Development Status :: 3 - Alpha', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", ], ) diff --git a/gyp/test_gyp.py b/gyp/test_gyp.py index 5975f8d4c6..382e75272d 100755 --- a/gyp/test_gyp.py +++ b/gyp/test_gyp.py @@ -229,8 +229,7 @@ def run_test(self, test, fmt, i): and not (stdout.endswith("NO RESULT\n")) ): print() - for l in stdout.splitlines(): - print(" %s" % l) + print("\n".join(" %s" % line for line in stdout.splitlines())) elif not self.isatty: print()