[infra] Update VS toolchain to stay up-to-date with flutter engine and chrome

Visual Studio is at 2017 Update 9 now.

Change-Id: Id577de7bf08e8ebedffbeec4c467f78b0f76adcd
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/108906
Reviewed-by: Ben Konyi <bkonyi@google.com>
Commit-Queue: Alexander Aprelev <aam@google.com>
This commit is contained in:
Alexander Aprelev 2019-07-31 20:01:01 +00:00 committed by commit-bot@chromium.org
parent 130c654dcd
commit 521a106332

View file

@ -7,10 +7,13 @@
# This file keeps the MSVC toolchain up-to-date for Google developers.
# It is copied from Chromium:
# https://cs.chromium.org/chromium/src/build/vs_toolchain.py
# with modifications that update paths, and remove dependencies on gyp.
# with modifications that update paths and remove dependencies on gyp.
# To update to a new MSVC toolchain, copy the updated script from the Chromium
# tree, and edit to make it work in the Dart tree by updating paths in the original script.
# tree, edit to make it work in the Dart tree by updating paths in the original script.
from __future__ import print_function
import collections
import glob
import json
import os
@ -31,17 +34,21 @@ SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(chrome_src, 'tools'))
json_data_file = os.path.join(script_dir, 'win_toolchain.json')
# Use MSVS2017 as the default toolchain.
CURRENT_DEFAULT_TOOLCHAIN_VERSION = '2017'
# VS versions are listed in descending order of priority (highest first).
MSVS_VERSIONS = collections.OrderedDict([
('2017', '15.0'),
('2019', '16.0'),
])
def SetEnvironmentAndGetRuntimeDllDirs():
"""Sets up os.environ to use the depot_tools VS toolchain with gyp, and
returns the location of the VS runtime DLLs so they can be copied into
returns the location of the VC runtime DLLs so they can be copied into
the output directory after gyp generation.
Return value is [x64path, x86path] or None
Return value is [x64path, x86path, 'Arm64Unused'] or None. arm64path is
generated separately because there are multiple folders for the arm64 VC
runtime.
"""
vs_runtime_dll_dirs = None
depot_tools_win_toolchain = \
@ -51,7 +58,10 @@ def SetEnvironmentAndGetRuntimeDllDirs():
if ((sys.platform in ('win32', 'cygwin') or os.path.exists(json_data_file))
and depot_tools_win_toolchain):
if ShouldUpdateToolchain():
update_result = Update()
if len(sys.argv) > 1 and sys.argv[1] == 'update':
update_result = Update()
else:
update_result = Update(no_download=True)
if update_result != 0:
raise Exception('Failed to update, error code %d.' % update_result)
with open(json_data_file, 'r') as tempf:
@ -97,7 +107,9 @@ def SetEnvironmentAndGetRuntimeDllDirs():
# don't build on ARM64 machines.
x64_path = 'System32' if bitness == '64bit' else 'Sysnative'
x64_path = os.path.join(os.path.expandvars('%windir%'), x64_path)
vs_runtime_dll_dirs = [x64_path, os.path.expandvars('%windir%/SysWOW64'),
vs_runtime_dll_dirs = [x64_path,
os.path.join(os.path.expandvars('%windir%'),
'SysWOW64'),
'Arm64Unused']
return vs_runtime_dll_dirs
@ -131,9 +143,36 @@ def _RegistryGetValue(key, value):
def GetVisualStudioVersion():
"""Return GYP_MSVS_VERSION of Visual Studio.
"""Return best available version of Visual Studio.
"""
return os.environ.get('GYP_MSVS_VERSION', CURRENT_DEFAULT_TOOLCHAIN_VERSION)
env_version = os.environ.get('GYP_MSVS_VERSION')
if env_version:
return env_version
supported_versions = MSVS_VERSIONS.keys()
# VS installed in depot_tools for Googlers
if bool(int(os.environ.get('DEPOT_TOOLS_WIN_TOOLCHAIN', '1'))):
return supported_versions[0]
# VS installed in system for external developers
supported_versions_str = ', '.join('{} ({})'.format(v,k)
for k,v in MSVS_VERSIONS.items())
available_versions = []
for version in supported_versions:
for path in (
os.environ.get('vs%s_install' % version),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s' % version)):
if path and os.path.exists(path):
available_versions.append(version)
break
if not available_versions:
raise Exception('No supported Visual Studio can be found.'
' Supported versions are: %s.' % supported_versions_str)
return available_versions[0]
def DetectVisualStudioPath():
@ -143,31 +182,30 @@ def DetectVisualStudioPath():
# Note that this code is used from
# build/toolchain/win/setup_toolchain.py as well.
version_as_year = GetVisualStudioVersion()
year_to_version = {
'2017': '15.0',
}
if version_as_year not in year_to_version:
raise Exception(('Visual Studio version %s (from GYP_MSVS_VERSION)'
' not supported. Supported versions are: %s') % (
version_as_year, ', '.join(year_to_version.keys())))
if version_as_year == '2017':
# The VC++ 2017 install location needs to be located using COM instead of
# the registry. For details see:
# https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/
# For now we use a hardcoded default with an environment variable override.
for path in (
os.environ.get('vs2017_install'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Enterprise'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Professional'),
os.path.expandvars('%ProgramFiles(x86)%'
'/Microsoft Visual Studio/2017/Community')):
if path and os.path.exists(path):
return path
raise Exception(('Visual Studio Version %s (from GYP_MSVS_VERSION)'
' not found.') % (version_as_year))
# The VC++ >=2017 install location needs to be located using COM instead of
# the registry. For details see:
# https://blogs.msdn.microsoft.com/heaths/2016/09/15/changes-to-visual-studio-15-setup/
# For now we use a hardcoded default with an environment variable override.
for path in (
os.environ.get('vs%s_install' % version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Enterprise' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Professional' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Community' %
version_as_year),
os.path.expandvars('%ProgramFiles(x86)%' +
'/Microsoft Visual Studio/%s/Preview' %
version_as_year)):
if path and os.path.exists(path):
return path
raise Exception('Visual Studio Version %s (from GYP_MSVS_VERSION)'
' not found.' % version_as_year)
def _CopyRuntimeImpl(target, source, verbose=True):
@ -180,7 +218,7 @@ def _CopyRuntimeImpl(target, source, verbose=True):
(not os.path.isfile(target) or
abs(os.stat(target).st_mtime - os.stat(source).st_mtime) >= 0.01)):
if verbose:
print 'Copying %s to %s...' % (source, target)
print('Copying %s to %s...' % (source, target))
if os.path.exists(target):
# Make the file writable so that we can delete it now, and keep it
# readable.
@ -191,10 +229,37 @@ def _CopyRuntimeImpl(target, source, verbose=True):
# keep it readable.
os.chmod(target, stat.S_IWRITE | stat.S_IREAD)
def _SortByHighestVersionNumberFirst(list_of_str_versions):
"""This sorts |list_of_str_versions| according to version number rules
so that version "1.12" is higher than version "1.9". Does not work
with non-numeric versions like 1.4.a8 which will be higher than
1.4.a12. It does handle the versions being embedded in file paths.
"""
def to_int_if_int(x):
try:
return int(x)
except ValueError:
return x
def to_number_sequence(x):
part_sequence = re.split(r'[\\/\.]', x)
return [to_int_if_int(x) for x in part_sequence]
list_of_str_versions.sort(key=to_number_sequence, reverse=True)
def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
"""Copy both the msvcp and vccorlib runtime DLLs, only if the target doesn't
exist, but the target directory does exist."""
if target_cpu == 'arm64':
# Windows ARM64 VCRuntime is located at {toolchain_root}/VC/Redist/MSVC/
# {x.y.z}/[debug_nonredist/]arm64/Microsoft.VC141.CRT/.
vc_redist_root = FindVCRedistRoot()
if suffix.startswith('.'):
source_dir = os.path.join(vc_redist_root,
'arm64', 'Microsoft.VC141.CRT')
else:
source_dir = os.path.join(vc_redist_root, 'debug_nonredist',
'arm64', 'Microsoft.VC141.DebugCRT')
for file_part in ('msvcp', 'vccorlib', 'vcruntime'):
dll = dll_pattern % file_part
target = os.path.join(target_dir, dll)
@ -212,8 +277,14 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
# ARM64 doesn't have a redist for the ucrt DLLs because they are always
# present in the OS.
if target_cpu != 'arm64':
ucrt_dll_dirs = os.path.join(win_sdk_dir, 'Redist', 'ucrt', 'DLLs',
target_cpu)
# Starting with the 10.0.17763 SDK the ucrt files are in a version-named
# directory - this handles both cases.
redist_dir = os.path.join(win_sdk_dir, 'Redist')
version_dirs = glob.glob(os.path.join(redist_dir, '10.*'))
if len(version_dirs) > 0:
_SortByHighestVersionNumberFirst(version_dirs)
redist_dir = version_dirs[0]
ucrt_dll_dirs = os.path.join(redist_dir, 'ucrt', 'DLLs', target_cpu)
ucrt_files = glob.glob(os.path.join(ucrt_dll_dirs, 'api-ms-win-*.dll'))
assert len(ucrt_files) > 0
for ucrt_src_file in ucrt_files:
@ -222,73 +293,60 @@ def _CopyUCRTRuntime(target_dir, source_dir, target_cpu, dll_pattern, suffix):
_CopyRuntimeImpl(ucrt_dst_file, ucrt_src_file, False)
# We must copy ucrtbase.dll for x64/x86, and ucrtbased.dll for all CPU types.
if target_cpu != 'arm64' or not suffix.startswith('.'):
if not suffix.startswith('.'):
# ucrtbased.dll is located at {win_sdk_dir}/bin/{a.b.c.d}/{target_cpu}/
# ucrt/.
sdk_redist_root = os.path.join(win_sdk_dir, 'bin')
sdk_bin_sub_dirs = os.listdir(sdk_redist_root)
# Select the most recent SDK if there are multiple versions installed.
_SortByHighestVersionNumberFirst(sdk_bin_sub_dirs)
for directory in sdk_bin_sub_dirs:
sdk_redist_root_version = os.path.join(sdk_redist_root, directory)
if not os.path.isdir(sdk_redist_root_version):
continue
if re.match(r'10\.\d+\.\d+\.\d+', directory):
source_dir = os.path.join(sdk_redist_root_version, target_cpu, 'ucrt')
break
_CopyRuntimeImpl(os.path.join(target_dir, 'ucrtbase' + suffix),
os.path.join(source_dir, 'ucrtbase' + suffix))
def FindVCToolsRoot():
"""In VS2017 the PGO runtime dependencies are located in
{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/Host{target_cpu}/{target_cpu}/, the
{version_number} part is likely to change in case of a minor update of the
toolchain so we don't hardcode this value here (except for the major number).
This returns the '{toolchain_root}/VC/Tools/MSVC/{x.y.z}/bin/' path.
This function should only be called when using VS2017.
def FindVCComponentRoot(component):
"""Find the most recent Tools or Redist or other directory in an MSVC install.
Typical results are {toolchain_root}/VC/{component}/MSVC/{x.y.z}. The {x.y.z}
version number part changes frequently so the highest version number found is
used.
"""
assert GetVisualStudioVersion() == '2017'
SetEnvironmentAndGetRuntimeDllDirs()
assert ('GYP_MSVS_OVERRIDE_PATH' in os.environ)
vc_tools_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'],
'VC', 'Tools', 'MSVC')
for directory in os.listdir(vc_tools_msvc_root):
if not os.path.isdir(os.path.join(vc_tools_msvc_root, directory)):
vc_component_msvc_root = os.path.join(os.environ['GYP_MSVS_OVERRIDE_PATH'],
'VC', component, 'MSVC')
vc_component_msvc_contents = os.listdir(vc_component_msvc_root)
# Select the most recent toolchain if there are several.
_SortByHighestVersionNumberFirst(vc_component_msvc_contents)
for directory in vc_component_msvc_contents:
if not os.path.isdir(os.path.join(vc_component_msvc_root, directory)):
continue
if re.match('14\.\d+\.\d+', directory):
return os.path.join(vc_tools_msvc_root, directory, 'bin')
raise Exception('Unable to find the VC tools directory.')
if re.match(r'14\.\d+\.\d+', directory):
return os.path.join(vc_component_msvc_root, directory)
raise Exception('Unable to find the VC %s directory.' % component)
def _CopyPGORuntime(target_dir, target_cpu):
"""Copy the runtime dependencies required during a PGO build.
def FindVCRedistRoot():
"""In >=VS2017, Redist binaries are located in
{toolchain_root}/VC/Redist/MSVC/{x.y.z}/{target_cpu}/.
This returns the '{toolchain_root}/VC/Redist/MSVC/{x.y.z}/' path.
"""
env_version = GetVisualStudioVersion()
# These dependencies will be in a different location depending on the version
# of the toolchain.
if env_version == '2017':
pgo_runtime_root = FindVCToolsRoot()
assert pgo_runtime_root
# There's no version of pgosweep.exe in HostX64/x86, so we use the copy
# from HostX86/x86.
pgo_x86_runtime_dir = os.path.join(pgo_runtime_root, 'HostX86', 'x86')
pgo_x64_runtime_dir = os.path.join(pgo_runtime_root, 'HostX64', 'x64')
pgo_arm64_runtime_dir = os.path.join(pgo_runtime_root, 'arm64')
else:
raise Exception('Unexpected toolchain version: %s.' % env_version)
# We need to copy 2 runtime dependencies used during the profiling step:
# - pgort140.dll: runtime library required to run the instrumented image.
# - pgosweep.exe: executable used to collect the profiling data
pgo_runtimes = ['pgort140.dll', 'pgosweep.exe']
for runtime in pgo_runtimes:
if target_cpu == 'x86':
source = os.path.join(pgo_x86_runtime_dir, runtime)
elif target_cpu == 'x64':
source = os.path.join(pgo_x64_runtime_dir, runtime)
elif target_cpu == 'arm64':
source = os.path.join(pgo_arm64_runtime_dir, runtime)
else:
raise NotImplementedError('Unexpected target_cpu value: ' + target_cpu)
if not os.path.exists(source):
raise Exception('Unable to find %s.' % source)
_CopyRuntimeImpl(os.path.join(target_dir, runtime), source)
return FindVCComponentRoot('Redist')
def _CopyRuntime(target_dir, source_dir, target_cpu, debug):
"""Copy the VS runtime DLLs, only if the target doesn't exist, but the target
directory does exist. Handles VS 2015 and VS 2017."""
directory does exist. Handles VS 2015, 2017 and 2019."""
suffix = 'd.dll' if debug else '.dll'
# VS 2017 uses the same CRT DLLs as VS 2015.
# VS 2015, 2017 and 2019 use the same CRT DLLs.
_CopyUCRTRuntime(target_dir, source_dir, target_cpu, '%s140' + suffix,
suffix)
@ -297,7 +355,7 @@ def CopyDlls(target_dir, configuration, target_cpu):
"""Copy the VS runtime DLLs into the requested directory as needed.
configuration is one of 'Debug' or 'Release'.
target_cpu is one of 'x86' or 'x64'.
target_cpu is one of 'x86', 'x64' or 'arm64'.
The debug configuration gets both the debug and release DLLs; the
release config only the latter.
@ -318,16 +376,13 @@ def CopyDlls(target_dir, configuration, target_cpu):
_CopyRuntime(target_dir, runtime_dir, target_cpu, debug=False)
if configuration == 'Debug':
_CopyRuntime(target_dir, runtime_dir, target_cpu, debug=True)
else:
_CopyPGORuntime(target_dir, target_cpu)
_CopyDebugger(target_dir, target_cpu)
def _CopyDebugger(target_dir, target_cpu):
"""Copy dbghelp.dll and dbgcore.dll into the requested directory as needed.
target_cpu is one of 'x86' or 'x64'.
target_cpu is one of 'x86', 'x64' or 'arm64'.
dbghelp.dll is used when Chrome needs to symbolize stacks. Copying this file
from the SDK directory avoids using the system copy of dbghelp.dll which then
@ -353,7 +408,7 @@ def _CopyDebugger(target_dir, target_cpu):
# TODO(crbug.com/773476): remove version requirement.
raise Exception('%s not found in "%s"\r\nYou must install the '
'"Debugging Tools for Windows" feature from the Windows'
' 10 SDK. You must use v10.0.17134.0. of the SDK'
' 10 SDK.'
% (debug_file, full_path))
target_path = os.path.join(target_dir, debug_file)
_CopyRuntimeImpl(target_path, full_path)
@ -364,9 +419,10 @@ def _GetDesiredVsToolchainHashes():
to build with."""
env_version = GetVisualStudioVersion()
if env_version == '2017':
# VS 2017 Update 7.1 (15.7.1) with 10.0.17134.12 SDK, rebuilt with
# dbghelp.dll fix.
toolchain_hash = '3bc0ec615cf20ee342f3bc29bc991b5ad66d8d2c'
# VS 2017 Update 9 (15.9.12) with 10.0.18362 SDK, 10.0.17763 version of
# Debuggers, and 10.0.17134 version of d3dcompiler_47.dll, with ARM64
# libraries.
toolchain_hash = '418b3076791776573a815eb298c8aa590307af63'
# Third parties that do not have access to the canonical toolchain can map
# canonical toolchain version to their own toolchain versions.
toolchain_hash_mapping_key = 'GYP_MSVS_HASH_%s' % toolchain_hash
@ -387,13 +443,15 @@ def ShouldUpdateToolchain():
return version != env_version
def Update(force=False):
def Update(force=False, no_download=False):
"""Requests an update of the toolchain to the specific hashes we have at
this revision. The update outputs a .json of the various configuration
information required to pass to gyp which we use in |GetToolchainDir()|.
If no_download is true then the toolchain will be configured if present but
will not be downloaded.
"""
if force != False and force != '--force':
print >>sys.stderr, 'Unknown parameter "%s"' % force
print('Unknown parameter "%s"' % force, file=sys.stderr)
return 1
if force == '--force' or os.path.exists(json_data_file):
force = True
@ -441,6 +499,8 @@ def Update(force=False):
] + _GetDesiredVsToolchainHashes()
if force:
get_toolchain_args.append('--force')
if no_download:
get_toolchain_args.append('--no-download')
subprocess.check_call(get_toolchain_args)
return 0
@ -473,17 +533,15 @@ def GetToolchainDir():
runtime_dll_dirs = SetEnvironmentAndGetRuntimeDllDirs()
win_sdk_dir = SetEnvironmentAndGetSDKDir()
print '''vs_path = %s
print('''vs_path = %s
sdk_path = %s
vs_version = %s
wdk_dir = %s
runtime_dirs = %s
''' % (
ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])),
ToGNString(win_sdk_dir),
ToGNString(GetVisualStudioVersion()),
ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))),
ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None'])))
''' % (ToGNString(NormalizePath(os.environ['GYP_MSVS_OVERRIDE_PATH'])),
ToGNString(win_sdk_dir), ToGNString(GetVisualStudioVersion()),
ToGNString(NormalizePath(os.environ.get('WDK_DIR', ''))),
ToGNString(os.path.pathsep.join(runtime_dll_dirs or ['None']))))
def main():
@ -493,10 +551,10 @@ def main():
'copy_dlls': CopyDlls,
}
if len(sys.argv) < 2 or sys.argv[1] not in commands:
print >>sys.stderr, 'Expected one of: %s' % ', '.join(commands)
print('Expected one of: %s' % ', '.join(commands), file=sys.stderr)
return 1
return commands[sys.argv[1]](*sys.argv[2:])
if __name__ == '__main__':
sys.exit(main())
sys.exit(main())