diff --git a/build/vs_toolchain.py b/build/vs_toolchain.py index 32eb669385e..c0f631affad 100644 --- a/build/vs_toolchain.py +++ b/build/vs_toolchain.py @@ -7,10 +7,13 @@ # This file keeps the MSVC toolchain up-to-date for Google developers. # It is copied from Chromium: # https://cs.chromium.org/chromium/src/build/vs_toolchain.py -# with modifications that update paths, and remove dependencies on gyp. +# with modifications that update paths and remove dependencies on gyp. # To update to a new MSVC toolchain, copy the updated script from the Chromium -# tree, and edit to make it work in the Dart tree by updating paths in the original script. +# tree, edit to make it work in the Dart tree by updating paths in the original script. +from __future__ import print_function + +import collections import glob import json import os @@ -31,17 +34,21 @@ SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(chrome_src, 'tools')) json_data_file = os.path.join(script_dir, 'win_toolchain.json') - -# Use MSVS2017 as the default toolchain. -CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2017' +# VS versions are listed in descending order of priority (highest first). +MSVS_VERSIONS = collections.OrderedDict([ + ('2017', '15.0'), + ('2019', '16.0'), +]) def SetEnvironmentAndGetRuntimeDllDirs(): """Sets up os.environ to use the depot_tools VS toolchain with gyp, and - returns the location of the VS runtime DLLs so they can be copied into + returns the location of the VC runtime DLLs so they can be copied into the output directory after gyp generation. - Return value is [x64path, x86path] or None + Return value is [x64path, x86path, 'Arm64Unused'] or None. arm64path is + generated separately because there are multiple folders for the arm64 VC + runtime. """ vs_runtime_dll_dirs = None depot_tools_win_toolchain = \ @@ -51,7 +58,10 @@ def SetEnvironmentAndGetRuntimeDllDirs(): if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file)) and depot_tools_win_toolchain): if ShouldUpdateToolchain(): - update_result = Update() + if len(sys.argv) > 1 and sys.argv[1] == 'update': + update_result = Update() + else: + update_result = Update(no_download=True) if update_result != 0: raise Exception('Failed to update, error code %d.' % update_result) with open(json_data_file, 'r') as tempf: @@ -97,7 +107,9 @@ def SetEnvironmentAndGetRuntimeDllDirs(): # don't build on ARM64 machines. x64_path = 'System32' if bitness == '64bit' else 'Sysnative' x64_path = os.path.join(os.path.expandvars('%windir%'), x64_path) - vs_runtime_dll_dirs = [x64_path, os.path.expandvars('%windir%/SysWOW64'), + vs_runtime_dll_dirs = [x64_path, + os.path.join(os.path.expandvars('%windir%'), + 'SysWOW64'), 'Arm64Unused'] return vs_runtime_dll_dirs @@ -131,9 +143,36 @@ def _RegistryGetValue(key, value): def GetVisualStudioVersion(): - """Return GYP_MSVS_VERSION of Visual Studio. + """Return best available version of Visual Studio. """ - return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION) + + env_version = os.environ.get('GYP_MSVS_VERSION') + if env_version: + return env_version + + supported_versions = MSVS_VERSIONS.keys() + + # VS installed in depot_tools for Googlers + if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))): + return supported_versions[0] + + # VS installed in system for external developers + supported_versions_str = ', '.join('{} ({})'.format(v,k) + for k,v in MSVS_VERSIONS.items()) + available_versions = [] + for version in supported_versions: + for path in ( + os.environ.get('vs%s_install' % version), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s' % version)): + if path and os.path.exists(path): + available_versions.append(version) + break + + if not available_versions: + raise Exception('No supported Visual Studio can be found.' + ' Supported versions are: %s.' % supported_versions_str) + return available_versions[0] def DetectVisualStudioPath(): @@ -143,31 +182,30 @@ def DetectVisualStudioPath(): # Note that this code is used from # build/toolchain/win/setup_toolchain.py as well. version_as_year = GetVisualStudioVersion() - year_to_version = { - '2017': '15.0', - } - if version_as_year not in year_to_version: - raise Exception(('Visual Studio version %s (from GYP_MSVS_VERSION)' - ' not supported. Supported versions are: %s') % ( - version_as_year, ', '.join(year_to_version.keys()))) - if version_as_year == '2017': - # The VC++ 2017 install location needs to be located using COM instead of - # the registry. For details see: - # https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/ - # For now we use a hardcoded default with an environment variable override. - for path in ( - os.environ.get('vs2017_install'), - os.path.expandvars('%ProgramFiles(x86)%' - '/Microsoft Visual Studio/2017/Enterprise'), - os.path.expandvars('%ProgramFiles(x86)%' - '/Microsoft Visual Studio/2017/Professional'), - os.path.expandvars('%ProgramFiles(x86)%' - '/Microsoft Visual Studio/2017/Community')): - if path and os.path.exists(path): - return path - raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)' - ' not found.') % (version_as_year)) + # The VC++ >=2017 install location needs to be located using COM instead of + # the registry. For details see: + # https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/ + # For now we use a hardcoded default with an environment variable override. + for path in ( + os.environ.get('vs%s_install' % version_as_year), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Enterprise' % + version_as_year), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Professional' % + version_as_year), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Community' % + version_as_year), + os.path.expandvars('%ProgramFiles(x86)%' + + '/Microsoft Visual Studio/%s/Preview' % + version_as_year)): + if path and os.path.exists(path): + return path + + raise Exception('Visual Studio Version %s (from GYP_MSVS_VERSION)' + ' not found.' % version_as_year) def _CopyRuntimeImpl(target, source, verbose=True): @@ -180,7 +218,7 @@ def _CopyRuntimeImpl(target, source, verbose=True): (not os.path.isfile(target) or abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)): if verbose: - print 'Copying %s to %s...' % (source, target) + print('Copying %s to %s...' % (source, target)) if os.path.exists(target): # Make the file writable so that we can delete it now, and keep it # readable. @@ -191,10 +229,37 @@ def _CopyRuntimeImpl(target, source, verbose=True): # keep it readable. os.chmod(target, stat.S_IWRITE | stat.S_IREAD) +def _SortByHighestVersionNumberFirst(list_of_str_versions): + """This sorts |list_of_str_versions| according to version number rules + so that version "1.12" is higher than version "1.9". Does not work + with non-numeric versions like 1.4.a8 which will be higher than + 1.4.a12. It does handle the versions being embedded in file paths. + """ + def to_int_if_int(x): + try: + return int(x) + except ValueError: + return x + + def to_number_sequence(x): + part_sequence = re.split(r'[\\/\.]', x) + return [to_int_if_int(x) for x in part_sequence] + + list_of_str_versions.sort(key=to_number_sequence, reverse=True) def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix): """Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't exist, but the target directory does exist.""" + if target_cpu == 'arm64': + # Windows ARM64 VCRuntime is located at {toolchain_root}/VC/Redist/MSVC/ + # {x.y.z}/[debug_nonredist/]arm64/Microsoft.VC141.CRT/. + vc_redist_root = FindVCRedistRoot() + if suffix.startswith('.'): + source_dir = os.path.join(vc_redist_root, + 'arm64', 'Microsoft.VC141.CRT') + else: + source_dir = os.path.join(vc_redist_root, 'debug_nonredist', + 'arm64', 'Microsoft.VC141.DebugCRT') for file_part in ('msvcp', 'vccorlib', 'vcruntime'): dll = dll_pattern % file_part target = os.path.join(target_dir, dll) @@ -212,8 +277,14 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix): # ARM64 doesn't have a redist for the ucrt DLLs because they are always # present in the OS. if target_cpu != 'arm64': - ucrt_dll_dirs = os.path.join(win_sdk_dir, 'Redist', 'ucrt', 'DLLs', - target_cpu) + # Starting with the 10.0.17763 SDK the ucrt files are in a version-named + # directory - this handles both cases. + redist_dir = os.path.join(win_sdk_dir, 'Redist') + version_dirs = glob.glob(os.path.join(redist_dir, '10.*')) + if len(version_dirs) > 0: + _SortByHighestVersionNumberFirst(version_dirs) + redist_dir = version_dirs[0] + ucrt_dll_dirs = os.path.join(redist_dir, 'ucrt', 'DLLs', target_cpu) ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll')) assert len(ucrt_files) > 0 for ucrt_src_file in ucrt_files: @@ -222,73 +293,60 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix): _CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False) # We must copy ucrtbase.dll for x64/x86, and ucrtbased.dll for all CPU types. if target_cpu != 'arm64' or not suffix.startswith('.'): + if not suffix.startswith('.'): + # ucrtbased.dll is located at {win_sdk_dir}/bin/{a.b.c.d}/{target_cpu}/ + # ucrt/. + sdk_redist_root = os.path.join(win_sdk_dir, 'bin') + sdk_bin_sub_dirs = os.listdir(sdk_redist_root) + # Select the most recent SDK if there are multiple versions installed. + _SortByHighestVersionNumberFirst(sdk_bin_sub_dirs) + for directory in sdk_bin_sub_dirs: + sdk_redist_root_version = os.path.join(sdk_redist_root, directory) + if not os.path.isdir(sdk_redist_root_version): + continue + if re.match(r'10\.\d+\.\d+\.\d+', directory): + source_dir = os.path.join(sdk_redist_root_version, target_cpu, 'ucrt') + break _CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix), os.path.join(source_dir, 'ucrtbase' + suffix)) -def FindVCToolsRoot(): - """In VS2017 the PGO runtime dependencies are located in - {toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/Host{target_cpu}/{target_cpu}/, the - {version_number} part is likely to change in case of a minor update of the - toolchain so we don't hardcode this value here (except for the major number). - - This returns the '{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/' path. - - This function should only be called when using VS2017. +def FindVCComponentRoot(component): + """Find the most recent Tools or Redist or other directory in an MSVC install. + Typical results are {toolchain_root}/VC/{component}/MSVC/{x.y.z}. The {x.y.z} + version number part changes frequently so the highest version number found is + used. """ - assert GetVisualStudioVersion() == '2017' + SetEnvironmentAndGetRuntimeDllDirs() assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ) - vc_tools_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], - 'VC', 'Tools', 'MSVC') - for directory in os.listdir(vc_tools_msvc_root): - if not os.path.isdir(os.path.join(vc_tools_msvc_root, directory)): + vc_component_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'], + 'VC', component, 'MSVC') + vc_component_msvc_contents = os.listdir(vc_component_msvc_root) + # Select the most recent toolchain if there are several. + _SortByHighestVersionNumberFirst(vc_component_msvc_contents) + for directory in vc_component_msvc_contents: + if not os.path.isdir(os.path.join(vc_component_msvc_root, directory)): continue - if re.match('14\.\d+\.\d+', directory): - return os.path.join(vc_tools_msvc_root, directory, 'bin') - raise Exception('Unable to find the VC tools directory.') + if re.match(r'14\.\d+\.\d+', directory): + return os.path.join(vc_component_msvc_root, directory) + raise Exception('Unable to find the VC %s directory.' % component) -def _CopyPGORuntime(target_dir, target_cpu): - """Copy the runtime dependencies required during a PGO build. +def FindVCRedistRoot(): + """In >=VS2017, Redist binaries are located in + {toolchain_root}/VC/Redist/MSVC/{x.y.z}/{target_cpu}/. + + This returns the '{toolchain_root}/VC/Redist/MSVC/{x.y.z}/' path. """ - env_version = GetVisualStudioVersion() - # These dependencies will be in a different location depending on the version - # of the toolchain. - if env_version == '2017': - pgo_runtime_root = FindVCToolsRoot() - assert pgo_runtime_root - # There's no version of pgosweep.exe in HostX64/x86, so we use the copy - # from HostX86/x86. - pgo_x86_runtime_dir = os.path.join(pgo_runtime_root, 'HostX86', 'x86') - pgo_x64_runtime_dir = os.path.join(pgo_runtime_root, 'HostX64', 'x64') - pgo_arm64_runtime_dir = os.path.join(pgo_runtime_root, 'arm64') - else: - raise Exception('Unexpected toolchain version: %s.' % env_version) - - # We need to copy 2 runtime dependencies used during the profiling step: - # - pgort140.dll: runtime library required to run the instrumented image. - # - pgosweep.exe: executable used to collect the profiling data - pgo_runtimes = ['pgort140.dll', 'pgosweep.exe'] - for runtime in pgo_runtimes: - if target_cpu == 'x86': - source = os.path.join(pgo_x86_runtime_dir, runtime) - elif target_cpu == 'x64': - source = os.path.join(pgo_x64_runtime_dir, runtime) - elif target_cpu == 'arm64': - source = os.path.join(pgo_arm64_runtime_dir, runtime) - else: - raise NotImplementedError('Unexpected target_cpu value: ' + target_cpu) - if not os.path.exists(source): - raise Exception('Unable to find %s.' % source) - _CopyRuntimeImpl(os.path.join(target_dir, runtime), source) + return FindVCComponentRoot('Redist') def _CopyRuntime(target_dir, source_dir, target_cpu, debug): """Copy the VS runtime DLLs, only if the target doesn't exist, but the target - directory does exist. Handles VS 2015 and VS 2017.""" + directory does exist. Handles VS 2015, 2017 and 2019.""" suffix = 'd.dll' if debug else '.dll' - # VS 2017 uses the same CRT DLLs as VS 2015. + # VS 2015, 2017 and 2019 use the same CRT DLLs. _CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix, suffix) @@ -297,7 +355,7 @@ def CopyDlls(target_dir, configuration, target_cpu): """Copy the VS runtime DLLs into the requested directory as needed. configuration is one of 'Debug' or 'Release'. - target_cpu is one of 'x86' or 'x64'. + target_cpu is one of 'x86', 'x64' or 'arm64'. The debug configuration gets both the debug and release DLLs; the release config only the latter. @@ -318,16 +376,13 @@ def CopyDlls(target_dir, configuration, target_cpu): _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False) if configuration == 'Debug': _CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True) - else: - _CopyPGORuntime(target_dir, target_cpu) - _CopyDebugger(target_dir, target_cpu) def _CopyDebugger(target_dir, target_cpu): """Copy dbghelp.dll and dbgcore.dll into the requested directory as needed. - target_cpu is one of 'x86' or 'x64'. + target_cpu is one of 'x86', 'x64' or 'arm64'. dbghelp.dll is used when Chrome needs to symbolize stacks. Copying this file from the SDK directory avoids using the system copy of dbghelp.dll which then @@ -353,7 +408,7 @@ def _CopyDebugger(target_dir, target_cpu): # TODO(crbug.com/773476): remove version requirement. raise Exception('%s not found in "%s"\r\nYou must install the ' '"Debugging Tools for Windows" feature from the Windows' - ' 10 SDK. You must use v10.0.17134.0. of the SDK' + ' 10 SDK.' % (debug_file, full_path)) target_path = os.path.join(target_dir, debug_file) _CopyRuntimeImpl(target_path, full_path) @@ -364,9 +419,10 @@ def _GetDesiredVsToolchainHashes(): to build with.""" env_version = GetVisualStudioVersion() if env_version == '2017': - # VS 2017 Update 7.1 (15.7.1) with 10.0.17134.12 SDK, rebuilt with - # dbghelp.dll fix. - toolchain_hash = '3bc0ec615cf20ee342f3bc29bc991b5ad66d8d2c' + # VS 2017 Update 9 (15.9.12) with 10.0.18362 SDK, 10.0.17763 version of + # Debuggers, and 10.0.17134 version of d3dcompiler_47.dll, with ARM64 + # libraries. + toolchain_hash = '418b3076791776573a815eb298c8aa590307af63' # Third parties that do not have access to the canonical toolchain can map # canonical toolchain version to their own toolchain versions. toolchain_hash_mapping_key = 'GYP_MSVS_HASH_%s' % toolchain_hash @@ -387,13 +443,15 @@ def ShouldUpdateToolchain(): return version != env_version -def Update(force=False): +def Update(force=False, no_download=False): """Requests an update of the toolchain to the specific hashes we have at this revision. The update outputs a .json of the various configuration information required to pass to gyp which we use in |GetToolchainDir()|. + If no_download is true then the toolchain will be configured if present but + will not be downloaded. """ if force != False and force != '--force': - print >>sys.stderr, 'Unknown parameter "%s"' % force + print('Unknown parameter "%s"' % force, file=sys.stderr) return 1 if force == '--force' or os.path.exists(json_data_file): force = True @@ -441,6 +499,8 @@ def Update(force=False): ] + _GetDesiredVsToolchainHashes() if force: get_toolchain_args.append('--force') + if no_download: + get_toolchain_args.append('--no-download') subprocess.check_call(get_toolchain_args) return 0 @@ -473,17 +533,15 @@ def GetToolchainDir(): runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs() win_sdk_dir = SetEnvironmentAndGetSDKDir() - print '''vs_path = %s + print('''vs_path = %s sdk_path = %s vs_version = %s wdk_dir = %s runtime_dirs = %s -''' % ( - ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])), - ToGNString(win_sdk_dir), - ToGNString(GetVisualStudioVersion()), - ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))), - ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None']))) +''' % (ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])), + ToGNString(win_sdk_dir), ToGNString(GetVisualStudioVersion()), + ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))), + ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None'])))) def main(): @@ -493,10 +551,10 @@ def main(): 'copy_dlls': CopyDlls, } if len(sys.argv) < 2 or sys.argv[1] not in commands: - print >>sys.stderr, 'Expected one of: %s' % ', '.join(commands) + print('Expected one of: %s' % ', '.join(commands), file=sys.stderr) return 1 return commands[sys.argv[1]](*sys.argv[2:]) if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) \ No newline at end of file