tools,gyp: add support for MSVC cross-compilation

This change means that GYP can now generate two sets of projects: one
exclusively for a host x64 machine and one containing a mix of x64 and
Arm targets. The names of host targets are fixed up to end with
_host.exe, and any actions involving them are fixed up. This allows
compilation of Node on an x64 server for a Windows on Arm target.

PR-URL: #32867
Reviewed-By: Christian Clauss <[email protected]>
Reviewed-By: João Reis <[email protected]>

Author: richard-townsend-arm

tools/gyp/pylib/gyp/generator/msvs.py

@@ -40,6 +40,7 @@
 # letters.
 VALID_MSVS_GUID_CHARS = re.compile(r'^[A-F0-9\-]+$')
 
+generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()
 
 generator_default_variables = {
     'DRIVER_PREFIX': '',
@@ -51,7 +52,7 @@
     'STATIC_LIB_SUFFIX': '.lib',
     'SHARED_LIB_SUFFIX': '.dll',
     'INTERMEDIATE_DIR': '$(IntDir)',
-    'SHARED_INTERMEDIATE_DIR': '$(OutDir)obj/global_intermediate',
+    'SHARED_INTERMEDIATE_DIR': '$(OutDir)/obj/global_intermediate',
     'OS': 'win',
     'PRODUCT_DIR': '$(OutDir)',
     'LIB_DIR': '$(OutDir)lib',
@@ -286,7 +287,7 @@ def _ConfigTargetVersion(config_data):
   return config_data.get('msvs_target_version', 'Windows7')
 
 
-def _ConfigPlatform(config_data):
+def _ConfigPlatform(config_data, spec):
   return config_data.get('msvs_configuration_platform', 'Win32')
 
 
@@ -297,8 +298,8 @@ def _ConfigBaseName(config_name, platform_name):
     return config_name
 
 
-def _ConfigFullName(config_name, config_data):
-  platform_name = _ConfigPlatform(config_data)
+def _ConfigFullName(config_name, config_data, spec):
+  platform_name = _ConfigPlatform(config_data, spec)
   return '%s|%s' % (_ConfigBaseName(config_name, platform_name), platform_name)
 
 
@@ -951,7 +952,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
   return toolset
 
 
-def _GenerateProject(project, options, version, generator_flags):
+def _GenerateProject(project, options, version, generator_flags, spec):
   """Generates a vcproj file.
 
   Arguments:
@@ -969,7 +970,7 @@ def _GenerateProject(project, options, version, generator_flags):
     return []
 
   if version.UsesVcxproj():
-    return _GenerateMSBuildProject(project, options, version, generator_flags)
+    return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
   else:
     return _GenerateMSVSProject(project, options, version, generator_flags)
 
@@ -1091,7 +1092,7 @@ def _GetUniquePlatforms(spec):
   # Gather list of unique platforms.
   platforms = OrderedSet()
   for configuration in spec['configurations']:
-    platforms.add(_ConfigPlatform(spec['configurations'][configuration]))
+    platforms.add(_ConfigPlatform(spec['configurations'][configuration], spec))
   platforms = list(platforms)
   return platforms
 
@@ -1801,6 +1802,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
   # Convert into a tree of dicts on path.
   for p in sln_projects:
     gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
+    if p.endswith("#host"):
+      target += "_host"
     gyp_dir = os.path.dirname(gyp_file)
     path_dict = _GetPathDict(root, gyp_dir)
     path_dict[target + '.vcproj'] = project_objects[p]
@@ -1819,7 +1822,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
   default_config = _GetDefaultConfiguration(spec)
   proj_filename = default_config.get('msvs_existing_vcproj')
   if not proj_filename:
-    proj_filename = (spec['target_name'] + options.suffix +
+    proj_filename = spec['target_name']
+    if spec['toolset'] == 'host':
+      proj_filename += "_host"
+    proj_filename = (proj_filename + options.suffix +
                      msvs_version.ProjectExtension())
 
   build_file = gyp.common.BuildFile(qualified_target)
@@ -1838,10 +1844,12 @@ def _GetPlatformOverridesOfProject(spec):
   # solution configurations for this target.
   config_platform_overrides = {}
   for config_name, c in spec['configurations'].items():
-    config_fullname = _ConfigFullName(config_name, c)
-    platform = c.get('msvs_target_platform', _ConfigPlatform(c))
+    config_fullname = _ConfigFullName(config_name, c, spec)
+    platform = c.get('msvs_target_platform', _ConfigPlatform(c, spec))
     fixed_config_fullname = '%s|%s' % (
-        _ConfigBaseName(config_name, _ConfigPlatform(c)), platform)
+        _ConfigBaseName(config_name, _ConfigPlatform(c, spec)), platform)
+    if spec['toolset'] == 'host' and generator_supports_multiple_toolsets:
+      fixed_config_fullname = '%s|x64' % (config_name,)
     config_platform_overrides[config_fullname] = fixed_config_fullname
   return config_platform_overrides
 
@@ -1862,19 +1870,18 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
   projects = {}
   for qualified_target in target_list:
     spec = target_dicts[qualified_target]
-    if spec['toolset'] != 'target':
-      raise GypError(
-          'Multiple toolsets not supported in msvs build (target %s)' %
-          qualified_target)
     proj_path, fixpath_prefix = _GetPathOfProject(qualified_target, spec,
                                                   options, msvs_version)
     guid = _GetGuidOfProject(proj_path, spec)
     overrides = _GetPlatformOverridesOfProject(spec)
     build_file = gyp.common.BuildFile(qualified_target)
     # Create object for this project.
+    target_name = spec['target_name']
+    if spec['toolset'] == 'host':
+      target_name += '_host'
     obj = MSVSNew.MSVSProject(
         proj_path,
-        name=spec['target_name'],
+        name=target_name,
         guid=guid,
         spec=spec,
         build_file=build_file,
@@ -2041,7 +2048,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
   for qualified_target in target_list:
     spec = target_dicts[qualified_target]
     for config_name, config in spec['configurations'].items():
-      configs.add(_ConfigFullName(config_name, config))
+      config_name = _ConfigFullName(config_name, config, spec)
+      configs.add(config_name)
+      if config_name == 'Release|arm64':
+        configs.add("Release|x64")
   configs = list(configs)
 
   # Figure out all the projects that will be generated and their guids
@@ -2053,11 +2063,14 @@ def GenerateOutput(target_list, target_dicts, data, params):
   for project in project_objects.values():
     fixpath_prefix = project.fixpath_prefix
     missing_sources.extend(_GenerateProject(project, options, msvs_version,
-                                            generator_flags))
+                                            generator_flags, spec))
   fixpath_prefix = None
 
   for build_file in data:
     # Validate build_file extension
+    target_only_configs = configs
+    if generator_supports_multiple_toolsets:
+      target_only_configs = [i for i in configs if i.endswith('arm64')]
     if not build_file.endswith('.gyp'):
       continue
     sln_path = os.path.splitext(build_file)[0] + options.suffix + '.sln'
@@ -2072,7 +2085,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
     # Create solution.
     sln = MSVSNew.MSVSSolution(sln_path,
                                entries=root_entries,
-                               variants=configs,
+                               variants=target_only_configs,
                                websiteProperties=False,
                                version=msvs_version)
     sln.Write()
@@ -2674,21 +2687,23 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
   easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)
 
 
-def _GetConfigurationAndPlatform(name, settings):
+def _GetConfigurationAndPlatform(name, settings, spec):
   configuration = name.rsplit('_', 1)[0]
   platform = settings.get('msvs_configuration_platform', 'Win32')
+  if spec['toolset'] == 'host' and platform == 'arm64':
+    platform = 'x64' # Host-only tools are always built for x64
   return (configuration, platform)
 
 
-def _GetConfigurationCondition(name, settings):
+def _GetConfigurationCondition(name, settings, spec):
   return (r"'$(Configuration)|$(Platform)'=='%s|%s'" %
-          _GetConfigurationAndPlatform(name, settings))
+          _GetConfigurationAndPlatform(name, settings, spec))
 
 
-def _GetMSBuildProjectConfigurations(configurations):
+def _GetMSBuildProjectConfigurations(configurations, spec):
   group = ['ItemGroup', {'Label': 'ProjectConfigurations'}]
   for (name, settings) in sorted(configurations.items()):
-    configuration, platform = _GetConfigurationAndPlatform(name, settings)
+    configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
     designation = '%s|%s' % (configuration, platform)
     group.append(
         ['ProjectConfiguration', {'Include': designation},
@@ -2740,7 +2755,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name):
   platform_name = None
   msvs_windows_sdk_version = None
   for configuration in spec['configurations'].values():
-    platform_name = platform_name or _ConfigPlatform(configuration)
+    platform_name = platform_name or _ConfigPlatform(configuration, spec)
     msvs_windows_sdk_version = (msvs_windows_sdk_version or
                   _ConfigWindowsTargetPlatformVersion(configuration, version))
     if platform_name and msvs_windows_sdk_version:
@@ -2762,7 +2777,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
   properties = {}
   for name, settings in spec['configurations'].items():
     msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
-    condition = _GetConfigurationCondition(name, settings)
+    condition = _GetConfigurationCondition(name, settings, spec)
     character_set = msbuild_attributes.get('CharacterSet')
     config_type = msbuild_attributes.get('ConfigurationType')
     _AddConditionalProperty(properties, condition, 'ConfigurationType',
@@ -2790,12 +2805,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
   return properties
 
 
-def _GetMSBuildPropertySheets(configurations):
+def _GetMSBuildPropertySheets(configurations, spec):
   user_props = r'$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props'
   additional_props = {}
   props_specified = False
   for name, settings in sorted(configurations.items()):
-    configuration = _GetConfigurationCondition(name, settings)
+    configuration = _GetConfigurationCondition(name, settings, spec)
     if 'msbuild_props' in settings:
       additional_props[configuration] = _FixPaths(settings['msbuild_props'])
       props_specified = True
@@ -2946,7 +2961,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):
 
   properties = {}
   for (name, configuration) in sorted(configurations.items()):
-    condition = _GetConfigurationCondition(name, configuration)
+    condition = _GetConfigurationCondition(name, configuration, spec)
     attributes = _GetMSBuildAttributes(spec, configuration, build_file)
     msbuild_settings = configuration['finalized_msbuild_settings']
     _AddConditionalProperty(properties, condition, 'IntDir',
@@ -3055,7 +3070,9 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
   for (name, configuration) in sorted(configurations.items()):
     msbuild_settings = configuration['finalized_msbuild_settings']
     group = ['ItemDefinitionGroup',
-             {'Condition': _GetConfigurationCondition(name, configuration)}
+             {'Condition':
+              _GetConfigurationCondition(name, configuration, spec)
+             }
             ]
     for tool_name, tool_settings in sorted(msbuild_settings.items()):
       # Skip the tool named '' which is a holder of global settings handled
@@ -3277,7 +3294,9 @@ def _AddSources2(spec, sources, exclusions, grouped_sources,
                 extensions_excluded_from_precompile = ['.c']
 
           if precompiled_source == source:
-            condition = _GetConfigurationCondition(config_name, configuration)
+            condition = _GetConfigurationCondition(
+              config_name, configuration, spec
+            )
             detail.append(['PrecompiledHeader',
                            {'Condition': condition},
                            'Create'
@@ -3296,12 +3315,26 @@ def _AddSources2(spec, sources, exclusions, grouped_sources,
                                                      _GetUniquePlatforms(spec))
         grouped_sources[group].append([element, {'Include': source}] + detail)
 
-
-def _GetMSBuildProjectReferences(project):
+def _GetMSBuildProjectReferences(project, spec):
+  current_configuration = spec['default_configuration']
   references = []
   if project.dependencies:
     group = ['ItemGroup']
+    added_dependency_set = set()
     for dependency in project.dependencies:
+      dependency_spec = dependency.spec
+      should_skip_dep = False
+      if project.spec["toolset"] == 'target':
+        if dependency_spec['toolset'] == 'host':
+          if dependency_spec['type'] == 'static_library':
+            should_skip_dep = True
+      if dependency.name.startswith('run_'):
+        should_skip_dep = False
+      if should_skip_dep:
+        continue
+
+      canonical_name = dependency.name.replace('_host', '')
+      added_dependency_set.add(canonical_name)
       guid = dependency.guid
       project_dir = os.path.split(project.path)[0]
       relative_path = gyp.common.RelativePath(dependency.path, project_dir)
@@ -3323,7 +3356,7 @@ def _GetMSBuildProjectReferences(project):
   return references
 
 
-def _GenerateMSBuildProject(project, options, version, generator_flags):
+def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
   spec = project.spec
   configurations = spec['configurations']
   project_dir, project_file_name = os.path.split(project.path)
@@ -3411,7 +3444,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
        'DefaultTargets': 'Build'
       }]
 
-  content += _GetMSBuildProjectConfigurations(configurations)
+  content += _GetMSBuildProjectConfigurations(configurations, spec)
   content += _GetMSBuildGlobalProperties(spec, version, project.guid,
                                          project_file_name)
   content += import_default_section
@@ -3422,18 +3455,18 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
    content += _GetMSBuildLocalProperties(project.msbuild_toolset)
   content += import_cpp_props_section
   content += import_masm_props_section
-  if spec.get('msvs_enable_marmasm'):
+  if spec.get('msvs_enable_marmasm') or True:
     content += import_marmasm_props_section
   content += _GetMSBuildExtensions(props_files_of_rules)
-  content += _GetMSBuildPropertySheets(configurations)
+  content += _GetMSBuildPropertySheets(configurations, spec)
   content += macro_section
   content += _GetMSBuildConfigurationGlobalProperties(spec, configurations,
                                                       project.build_file)
   content += _GetMSBuildToolSettingsSections(spec, configurations)
   content += _GetMSBuildSources(
       spec, sources, exclusions, rule_dependencies, extension_to_rule_name,
       actions_spec, sources_handled_by_action, list_excluded)
-  content += _GetMSBuildProjectReferences(project)
+  content += _GetMSBuildProjectReferences(project, spec)
   content += import_cpp_targets_section
   content += import_masm_targets_section
   if spec.get('msvs_enable_marmasm'):
@@ -3516,15 +3549,25 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
   sources_handled_by_action = OrderedSet()
   actions_spec = []
   for primary_input, actions in actions_to_add.items():
+    if generator_supports_multiple_toolsets:
+      primary_input = primary_input.replace(".exe", "_host.exe")
     inputs = OrderedSet()
     outputs = OrderedSet()
     descriptions = []
     commands = []
     for action in actions:
+      def fixup_host_exe(i):
+        if "$(OutDir)" in i:
+          i = i.replace('.exe', '_host.exe')
+        return i
+      if generator_supports_multiple_toolsets:
+        action['inputs'] = [fixup_host_exe(i) for i in action['inputs']]
       inputs.update(OrderedSet(action['inputs']))
       outputs.update(OrderedSet(action['outputs']))
       descriptions.append(action['description'])
       cmd = action['command']
+      if generator_supports_multiple_toolsets:
+        cmd = cmd.replace('.exe', "_host.exe")
       # For most actions, add 'call' so that actions that invoke batch files
       # return and continue executing.  msbuild_use_call provides a way to
       # disable this but I have not seen any adverse effect from doing that