gyp: sync code base with nodejs repo (#1975)

PR-URL: #1975
Reviewed-By: Ujjwal Sharma <ryzokuken@disroot.org>
Reviewed-By: Christian Clauss <cclauss@me.com>
Reviewed-By: Rod Vagg <rod@vagg.org>

gyp/AUTHORS

@@ -11,3 +11,5 @@ Ryan Norton <rnorton10@gmail.com>
 David J. Sankel <david@sankelsoftware.com>
 Eric N. Vander Weele <ericvw@gmail.com>
 Tom Freudenberg <th.freudenberg@gmail.com>
+Julien Brianceau <jbriance@cisco.com>
+Refael Ackermann <refack@gmail.com>

gyp/README.md

@@ -0,0 +1,4 @@
+GYP can Generate Your Projects.
+===================================
+
+Documents are available at [gyp.gsrc.io](https://gyp.gsrc.io), or you can check out ```md-pages``` branch to read those documents offline.

gyp/pylib/gyp/MSVSNew.py

@@ -7,6 +7,7 @@
 import hashlib
 import os
 import random
+from operator import attrgetter
 
 import gyp.common
 
@@ -59,9 +60,6 @@ def __cmp__(self, other):
     # Sort by name then guid (so things are in order on vs2008).
     return cmp((self.name, self.get_guid()), (other.name, other.get_guid()))
 
-  def __lt__(self, other):
-    return self.__cmp__(other) < 0
-
 
 class MSVSFolder(MSVSSolutionEntry):
   """Folder in a Visual Studio project or solution."""
@@ -89,7 +87,7 @@ def __init__(self, path, name = None, entries = None,
     self.guid = guid
 
     # Copy passed lists (or set to empty lists)
-    self.entries = sorted(list(entries or []))
+    self.entries = sorted(entries or [], key=attrgetter('path'))
     self.items = list(items or [])
 
     self.entry_type_guid = ENTRY_TYPE_GUIDS['folder']
@@ -233,7 +231,7 @@ def Write(self, writer=gyp.common.WriteOnDiff):
       if isinstance(e, MSVSFolder):
         entries_to_check += e.entries
 
-    all_entries = sorted(all_entries)
+    all_entries = sorted(all_entries, key=attrgetter('path'))
 
     # Open file and print header
     f = writer(self.path)

gyp/pylib/gyp/MSVSSettings.py

@@ -467,7 +467,7 @@ def ConvertToMSBuildSettings(msvs_settings, stderr=sys.stderr):
             msvs_tool[msvs_setting](msvs_value, msbuild_settings)
           except ValueError as e:
             print('Warning: while converting %s/%s to MSBuild, '
-                              '%s' % (msvs_tool_name, msvs_setting, e), file=stderr)
+                  '%s' % (msvs_tool_name, msvs_setting, e), file=stderr)
         else:
           _ValidateExclusionSetting(msvs_setting,
                                     msvs_tool,
@@ -477,7 +477,7 @@ def ConvertToMSBuildSettings(msvs_settings, stderr=sys.stderr):
                                     stderr)
     else:
       print('Warning: unrecognized tool %s while converting to '
-                        'MSBuild.' % msvs_tool_name, file=stderr)
+            'MSBuild.' % msvs_tool_name, file=stderr)
   return msbuild_settings
 
 
@@ -598,6 +598,7 @@ def _ValidateSettings(validators, settings, stderr):
 _Same(_compile, 'UseFullPaths', _boolean)  # /FC
 _Same(_compile, 'WholeProgramOptimization', _boolean)  # /GL
 _Same(_compile, 'XMLDocumentationFileName', _file_name)
+_Same(_compile, 'CompileAsWinRT', _boolean)  # /ZW
 
 _Same(_compile, 'AssemblerOutput',
       _Enumeration(['NoListing',

gyp/pylib/gyp/MSVSSettings_test.py

@@ -6,14 +6,14 @@
 
 """Unit tests for the MSVSSettings.py file."""
 
-try:
-  from cStringIO import StringIO
-except ImportError:
-  from io import StringIO
-
 import unittest
 import gyp.MSVSSettings as MSVSSettings
 
+try:
+  from StringIO import StringIO  # Python 2
+except ImportError:
+  from io import StringIO  # Python 3
+
 
 class TestSequenceFunctions(unittest.TestCase):
 

gyp/pylib/gyp/MSVSUtil.py

@@ -14,6 +14,7 @@
   'loadable_module': 'dll',
   'shared_library': 'dll',
   'static_library': 'lib',
+  'windows_driver': 'sys',
 }
 
 
@@ -110,7 +111,7 @@ def ShardTargets(target_list, target_dicts):
     else:
       new_target_dicts[t] = target_dicts[t]
   # Shard dependencies.
-  for t in new_target_dicts:
+  for t in sorted(new_target_dicts):
     for deptype in ('dependencies', 'dependencies_original'):
       dependencies = copy.copy(new_target_dicts[t].get(deptype, []))
       new_dependencies = []

gyp/pylib/gyp/MSVSVersion.py

@@ -15,12 +15,16 @@
 PY3 = bytes != str
 
 
+def JoinPath(*args):
+  return os.path.normpath(os.path.join(*args))
+
+
 class VisualStudioVersion(object):
   """Information regarding a version of Visual Studio."""
 
   def __init__(self, short_name, description,
                solution_version, project_version, flat_sln, uses_vcxproj,
-               path, sdk_based, default_toolset=None):
+               path, sdk_based, default_toolset=None, compatible_sdks=None):
     self.short_name = short_name
     self.description = description
     self.solution_version = solution_version
@@ -30,6 +34,9 @@ def __init__(self, short_name, description,
     self.path = path
     self.sdk_based = sdk_based
     self.default_toolset = default_toolset
+    compatible_sdks = compatible_sdks or []
+    compatible_sdks.sort(key=lambda v: float(v.replace('v', '')), reverse=True)
+    self.compatible_sdks = compatible_sdks
 
   def ShortName(self):
     return self.short_name
@@ -70,43 +77,67 @@ def DefaultToolset(self):
     of a user override."""
     return self.default_toolset
 
-  def SetupScript(self, target_arch):
+
+  def _SetupScriptInternal(self, target_arch):
     """Returns a command (with arguments) to be used to set up the
     environment."""
-    # Check if we are running in the SDK command line environment and use
-    # the setup script from the SDK if so. |target_arch| should be either
-    # 'x86' or 'x64'.
-    assert target_arch in ('x86', 'x64')
-    sdk_dir = os.environ.get('WindowsSDKDir')
-    if self.sdk_based and sdk_dir:
-      return [os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd')),
-              '/' + target_arch]
-    else:
-      # We don't use VC/vcvarsall.bat for x86 because vcvarsall calls
-      # vcvars32, which it can only find if VS??COMNTOOLS is set, which it
-      # isn't always.
-      if target_arch == 'x86':
-        if self.short_name >= '2013' and self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          # VS2013 and later, non-Express have a x64-x86 cross that we want
-          # to prefer.
-          return [os.path.normpath(
-             os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86']
-        # Otherwise, the standard x86 compiler.
-        return [os.path.normpath(
-          os.path.join(self.path, 'Common7/Tools/vsvars32.bat'))]
+    assert target_arch in ('x86', 'x64'), "target_arch not supported"
+    # If WindowsSDKDir is set and SetEnv.Cmd exists then we are using the
+    # depot_tools build tools and should run SetEnv.Cmd to set up the
+    # environment. The check for WindowsSDKDir alone is not sufficient because
+    # this is set by running vcvarsall.bat.
+    sdk_dir = os.environ.get('WindowsSDKDir', '')
+    setup_path = JoinPath(sdk_dir, 'Bin', 'SetEnv.Cmd')
+    if self.sdk_based and sdk_dir and os.path.exists(setup_path):
+      return [setup_path, '/' + target_arch]
+
+    is_host_arch_x64 = (
+      os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
+      os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'
+    )
+
+    # For VS2017 (and newer) it's fairly easy
+    if self.short_name >= '2017':
+      script_path = JoinPath(self.path,
+                             'VC', 'Auxiliary', 'Build', 'vcvarsall.bat')
+
+      # Always use a native executable, cross-compiling if necessary.
+      host_arch = 'amd64' if is_host_arch_x64 else 'x86'
+      msvc_target_arch = 'amd64' if target_arch == 'x64' else 'x86'
+      arg = host_arch
+      if host_arch != msvc_target_arch:
+        arg += '_' + msvc_target_arch
+
+      return [script_path, arg]
+
+    # We try to find the best version of the env setup batch.
+    vcvarsall = JoinPath(self.path, 'VC', 'vcvarsall.bat')
+    if target_arch == 'x86':
+      if self.short_name >= '2013' and self.short_name[-1] != 'e' and \
+         is_host_arch_x64:
+        # VS2013 and later, non-Express have a x64-x86 cross that we want
+        # to prefer.
+        return [vcvarsall, 'amd64_x86']
       else:
-        assert target_arch == 'x64'
-        arg = 'x86_amd64'
-        # Use the 64-on-64 compiler if we're not using an express
-        # edition and we're running on a 64bit OS.
-        if self.short_name[-1] != 'e' and (
-            os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
-            os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
-          arg = 'amd64'
-        return [os.path.normpath(
-            os.path.join(self.path, 'VC/vcvarsall.bat')), arg]
+        # Otherwise, the standard x86 compiler. We don't use VC/vcvarsall.bat
+        # for x86 because vcvarsall calls vcvars32, which it can only find if
+        # VS??COMNTOOLS is set, which isn't guaranteed.
+        return [JoinPath(self.path, 'Common7', 'Tools', 'vsvars32.bat')]
+    elif target_arch == 'x64':
+      arg = 'x86_amd64'
+      # Use the 64-on-64 compiler if we're not using an express edition and
+      # we're running on a 64bit OS.
+      if self.short_name[-1] != 'e' and is_host_arch_x64:
+        arg = 'amd64'
+      return [vcvarsall, arg]
+
+  def SetupScript(self, target_arch):
+    script_data = self._SetupScriptInternal(target_arch)
+    script_path = script_data[0]
+    if not os.path.exists(script_path):
+      raise Exception('%s is missing - make sure VC++ tools are installed.' %
+                      script_path)
+    return script_data
 
 
 def _RegistryQueryBase(sysdir, key, value):
@@ -181,11 +212,11 @@ def _RegistryGetValueUsingWinReg(key, value):
     ImportError if _winreg is unavailable.
   """
   try:
-      # Python 2
-      from _winreg import HKEY_LOCAL_MACHINE, OpenKey, QueryValueEx
+    # Python 2
+    from _winreg import HKEY_LOCAL_MACHINE, OpenKey, QueryValueEx
   except ImportError:
-      # Python 3
-      from winreg import HKEY_LOCAL_MACHINE, OpenKey, QueryValueEx
+    # Python 3
+    from winreg import HKEY_LOCAL_MACHINE, OpenKey, QueryValueEx
 
   try:
     root, subkey = key.split('\\', 1)
@@ -236,6 +267,26 @@ def _CreateVersion(name, path, sdk_based=False):
   if path:
     path = os.path.normpath(path)
   versions = {
+      '2019': VisualStudioVersion('2019',
+                                  'Visual Studio 2019',
+                                  solution_version='12.00',
+                                  project_version='16.0',
+                                  flat_sln=False,
+                                  uses_vcxproj=True,
+                                  path=path,
+                                  sdk_based=sdk_based,
+                                  default_toolset='v142',
+                                  compatible_sdks=['v8.1', 'v10.0']),
+      '2017': VisualStudioVersion('2017',
+                                  'Visual Studio 2017',
+                                  solution_version='12.00',
+                                  project_version='15.0',
+                                  flat_sln=False,
+                                  uses_vcxproj=True,
+                                  path=path,
+                                  sdk_based=sdk_based,
+                                  default_toolset='v141',
+                                  compatible_sdks=['v8.1', 'v10.0']),
       '2015': VisualStudioVersion('2015',
                                   'Visual Studio 2015',
                                   solution_version='12.00',
@@ -350,14 +401,15 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
     A list of visual studio versions installed in descending order of
     usage preference.
     Base this on the registry and a quick check if devenv.exe exists.
-    Only versions 8-10 are considered.
     Possibilities are:
       2005(e) - Visual Studio 2005 (8)
       2008(e) - Visual Studio 2008 (9)
       2010(e) - Visual Studio 2010 (10)
       2012(e) - Visual Studio 2012 (11)
       2013(e) - Visual Studio 2013 (12)
       2015    - Visual Studio 2015 (14)
+      2017    - Visual Studio 2017 (15)
+      2019    - Visual Studio 2019 (16)
     Where (e) is e for express editions of MSVS and blank otherwise.
   """
   version_to_year = {
@@ -367,6 +419,8 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
       '11.0': '2012',
       '12.0': '2013',
       '14.0': '2015',
+      '15.0': '2017',
+      '16.0': '2019',
   }
   versions = []
   for version in versions_to_check:
@@ -397,13 +451,18 @@ def _DetectVisualStudioVersions(versions_to_check, force_express):
 
     # The old method above does not work when only SDK is installed.
     keys = [r'HKLM\Software\Microsoft\VisualStudio\SxS\VC7',
-            r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\SxS\VC7']
+            r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\SxS\VC7',
+            r'HKLM\Software\Microsoft\VisualStudio\SxS\VS7',
+            r'HKLM\Software\Wow6432Node\Microsoft\VisualStudio\SxS\VS7']
     for index in range(len(keys)):
       path = _RegistryGetValue(keys[index], version)
       if not path:
         continue
       path = _ConvertToCygpath(path)
-      if version != '14.0':  # There is no Express edition for 2015.
+      if version == '15.0':
+          if os.path.exists(path):
+              versions.append(_CreateVersion('2017', path))
+      elif version != '14.0':  # There is no Express edition for 2015.
         versions.append(_CreateVersion(version_to_year[version] + 'e',
             os.path.join(path, '..'), sdk_based=True))
 
@@ -422,7 +481,7 @@ def SelectVisualStudioVersion(version='auto', allow_fallback=True):
   if version == 'auto':
     version = os.environ.get('GYP_MSVS_VERSION', 'auto')
   version_map = {
-    'auto': ('14.0', '12.0', '10.0', '9.0', '8.0', '11.0'),
+    'auto': ('16.0', '15.0', '14.0', '12.0', '10.0', '9.0', '8.0', '11.0'),
     '2005': ('8.0',),
     '2005e': ('8.0',),
     '2008': ('9.0',),
@@ -434,6 +493,8 @@ def SelectVisualStudioVersion(version='auto', allow_fallback=True):
     '2013': ('12.0',),
     '2013e': ('12.0',),
     '2015': ('14.0',),
+    '2017': ('15.0',),
+    '2019': ('16.0',),
   }
   override_path = os.environ.get('GYP_MSVS_OVERRIDE_PATH')
   if override_path: