# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. # This file contains a set of utilities functions used by other Python-based # scripts. import contextlib import datetime import glob import imp import json import os import platform import re import shutil import subprocess import sys import tarfile import tempfile import uuid try: # Not available on Windows. import resource except: pass DART_DIR = os.path.abspath(os.path.normpath(os.path.join(__file__, '..', '..'))) def GetBotUtils(): '''Dynamically load the tools/bots/bot_utils.py python module.''' return imp.load_source( 'bot_utils', os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py')) def GetMinidumpUtils(): '''Dynamically load the tools/minidump.py python module.''' return imp.load_source('minidump', os.path.join(DART_DIR, 'tools', 'minidump.py')) class Version(object): def __init__(self, channel, major, minor, patch, prerelease, prerelease_patch, abi_version, oldest_supported_abi_version): self.channel = channel self.major = major self.minor = minor self.patch = patch self.prerelease = prerelease self.prerelease_patch = prerelease_patch self.abi_version = abi_version self.oldest_supported_abi_version = oldest_supported_abi_version # Try to guess the host operating system. def GuessOS(): os_id = platform.system() if os_id == "Linux": return "linux" elif os_id == "Darwin": return "macos" elif os_id == "Windows" or os_id == "Microsoft": # On Windows Vista platform.system() can return "Microsoft" with some # versions of Python, see http://bugs.python.org/issue1082 for details. return "win32" elif os_id == 'FreeBSD': return 'freebsd' elif os_id == 'OpenBSD': return 'openbsd' elif os_id == 'SunOS': return 'solaris' else: return None # Try to guess the host architecture. def GuessArchitecture(): os_id = platform.machine() if os_id.startswith('armv5te'): return 'armv5te' elif os_id.startswith('armv6'): return 'armv6' elif os_id.startswith('arm'): return 'arm' elif os_id.startswith('aarch64'): return 'arm64' elif '64' in os_id: return 'x64' elif (not os_id) or (not re.match('(x|i[3-6])86', os_id) is None): return 'ia32' elif os_id == 'i86pc': return 'ia32' else: guess_os = GuessOS() print ("Warning: Guessing architecture %s based on os %s\n"\ % (os_id, guess_os)) if guess_os == 'win32': return 'ia32' return None # Try to guess the number of cpus on this machine. def GuessCpus(): if os.getenv("DART_NUMBER_OF_CORES") is not None: return int(os.getenv("DART_NUMBER_OF_CORES")) if os.path.exists("/proc/cpuinfo"): return int( subprocess.check_output( "grep -E '^processor' /proc/cpuinfo | wc -l", shell=True)) if os.path.exists("/usr/bin/hostinfo"): return int( subprocess.check_output( '/usr/bin/hostinfo |' ' grep "processors are logically available." |' ' awk "{ print \$1 }"', shell=True)) win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") if win_cpu_count: return int(win_cpu_count) return 2 def GetWindowsRegistryKeyName(name): import win32process # Check if python process is 64-bit or if it's 32-bit running in 64-bit OS. # We need to know whether host is 64-bit so that we are looking in right # registry for Visual Studio path. if sys.maxsize > 2**32 or win32process.IsWow64Process(): wow6432Node = 'Wow6432Node\\' else: wow6432Node = '' return r'SOFTWARE\%s%s' % (wow6432Node, name) # Try to guess Visual Studio location when buiding on Windows. def GuessVisualStudioPath(): defaultPath = r"C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7" \ r"\IDE" defaultExecutable = "devenv.com" if not IsWindows(): return defaultPath, defaultExecutable keyNamesAndExecutables = [ # Pair for non-Express editions. (GetWindowsRegistryKeyName(r'Microsoft\VisualStudio'), 'devenv.com'), # Pair for 2012 Express edition. (GetWindowsRegistryKeyName(r'Microsoft\VSWinExpress'), 'VSWinExpress.exe'), # Pair for pre-2012 Express editions. (GetWindowsRegistryKeyName(r'Microsoft\VCExpress'), 'VCExpress.exe') ] bestGuess = (0.0, (defaultPath, defaultExecutable)) import _winreg for (keyName, executable) in keyNamesAndExecutables: try: key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, keyName) except WindowsError: # Can't find this key - moving on the next one. continue try: subkeyCounter = 0 while True: try: subkeyName = _winreg.EnumKey(key, subkeyCounter) subkeyCounter += 1 except WindowsError: # Reached end of enumeration. Moving on the next key. break match = re.match(r'^\d+\.\d+$', subkeyName) if match: with _winreg.OpenKey(key, subkeyName) as subkey: try: (installDir, registrytype) = _winreg.QueryValueEx( subkey, 'InstallDir') except WindowsError: # Can't find value under the key - continue to the next key. continue isExpress = executable != 'devenv.com' if not isExpress and subkeyName == '14.0': # Stop search since if we found non-Express VS2015 version # installed, which is preferred version. return installDir, executable else: version = float(subkeyName) # We prefer higher version of Visual Studio and given equal # version numbers we prefer non-Express edition. if version > bestGuess[0]: bestGuess = (version, (installDir, executable)) finally: _winreg.CloseKey(key) return bestGuess[1] # Returns true if we're running under Windows. def IsWindows(): return GuessOS() == 'win32' # Reads a text file into an array of strings - one for each # line. Strips comments in the process. def ReadLinesFrom(name): result = [] for line in open(name): if '#' in line: line = line[:line.find('#')] line = line.strip() if len(line) == 0: continue result.append(line) return result # Filters out all arguments until the next '--' argument # occurs. def ListArgCallback(option, value, parser): if value is None: value = [] for arg in parser.rargs: if arg[:2].startswith('--'): break value.append(arg) del parser.rargs[:len(value)] setattr(parser.values, option.dest, value) # Filters out all argument until the first non '-' or the # '--' argument occurs. def ListDartArgCallback(option, value, parser): if value is None: value = [] for arg in parser.rargs: if arg[:2].startswith('--') or arg[0] != '-': break value.append(arg) del parser.rargs[:len(value)] setattr(parser.values, option.dest, value) # Mapping table between build mode and build configuration. BUILD_MODES = { 'debug': 'Debug', 'release': 'Release', 'product': 'Product', } # Mapping table between OS and build output location. BUILD_ROOT = { 'win32': os.path.join('out'), 'linux': os.path.join('out'), 'freebsd': os.path.join('out'), 'macos': os.path.join('xcodebuild'), } ARCH_FAMILY = { 'ia32': 'ia32', 'x64': 'ia32', 'arm': 'arm', 'armv6': 'arm', 'armv5te': 'arm', 'arm64': 'arm', 'simarm': 'ia32', 'simarmv6': 'ia32', 'simarmv5te': 'ia32', 'simarm64': 'ia32', 'simdbc': 'ia32', 'simdbc64': 'ia32', 'armsimdbc': 'arm', 'armsimdbc64': 'arm', 'simarm_x64': 'ia32', } ARCH_GUESS = GuessArchitecture() BASE_DIR = os.path.abspath(os.path.join(os.curdir, '..')) DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..')) VERSION_FILE = os.path.join(DART_DIR, 'tools', 'VERSION') def GetBuildbotGSUtilPath(): gsutil = '/b/build/scripts/slave/gsutil' if platform.system() == 'Windows': gsutil = 'e:\\\\b\\build\\scripts\\slave\\gsutil' return gsutil def GetBuildMode(mode): return BUILD_MODES[mode] def GetArchFamily(arch): return ARCH_FAMILY[arch] def IsCrossBuild(target_os, arch): host_arch = ARCH_GUESS return ((GetArchFamily(host_arch) != GetArchFamily(arch)) or (target_os != GuessOS())) def GetBuildConf(mode, arch, conf_os=None): if conf_os == 'android': return '%s%s%s' % (GetBuildMode(mode), conf_os.title(), arch.upper()) else: # Ask for a cross build if the host and target architectures don't match. host_arch = ARCH_GUESS cross_build = '' if GetArchFamily(host_arch) != GetArchFamily(arch): cross_build = 'X' return '%s%s%s' % (GetBuildMode(mode), cross_build, arch.upper()) def GetBuildDir(host_os): return BUILD_ROOT[host_os] def GetBuildRoot(host_os, mode=None, arch=None, target_os=None): build_root = GetBuildDir(host_os) if mode: build_root = os.path.join(build_root, GetBuildConf( mode, arch, target_os)) return build_root def GetBuildSdkBin(host_os, mode=None, arch=None, target_os=None): build_root = GetBuildRoot(host_os, mode, arch, target_os) return os.path.join(build_root, 'dart-sdk', 'bin') def GetBaseDir(): return BASE_DIR def GetShortVersion(): version = ReadVersionFile() return ('%s.%s.%s.%s.%s' % (version.major, version.minor, version.patch, version.prerelease, version.prerelease_patch)) def GetSemanticSDKVersion(no_git_hash=False, version_file=None): version = ReadVersionFile(version_file) if not version: return None if version.channel == 'be': postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision() elif version.channel == 'dev': postfix = '-dev.%s.%s' % (version.prerelease, version.prerelease_patch) else: assert version.channel == 'stable' postfix = '' return '%s.%s.%s%s' % (version.major, version.minor, version.patch, postfix) def GetVersion(no_git_hash=False, version_file=None): return GetSemanticSDKVersion(no_git_hash, version_file) # The editor used to produce the VERSION file put on gcs. We now produce this # in the bots archiving the sdk. # The content looks like this: #{ # "date": "2015-05-28", # "version": "1.11.0-edge.131653", # "revision": "535394c2657ede445142d8a92486d3899bbf49b5" #} def GetVersionFileContent(): result = { "date": str(datetime.date.today()), "version": GetVersion(), "revision": GetGitRevision() } return json.dumps(result, indent=2) def GetChannel(): version = ReadVersionFile() return version.channel def GetUserName(): key = 'USER' if sys.platform == 'win32': key = 'USERNAME' return os.environ.get(key, '') def GetAbiVersion(version_file=None): version = ReadVersionFile(version_file) return version.abi_version def GetOldestSupportedAbiVersion(version_file=None): version = ReadVersionFile(version_file) return version.oldest_supported_abi_version def ReadVersionFile(version_file=None): def match_against(pattern, file_content): match = re.search(pattern, file_content, flags=re.MULTILINE) if match: return match.group(1) return None if version_file == None: version_file = VERSION_FILE try: fd = open(version_file) content = fd.read() fd.close() except: print("Warning: Couldn't read VERSION file (%s)" % version_file) return None channel = match_against('^CHANNEL ([A-Za-z0-9]+)$', content) major = match_against('^MAJOR (\d+)$', content) minor = match_against('^MINOR (\d+)$', content) patch = match_against('^PATCH (\d+)$', content) prerelease = match_against('^PRERELEASE (\d+)$', content) prerelease_patch = match_against('^PRERELEASE_PATCH (\d+)$', content) abi_version = match_against('^ABI_VERSION (\d+)$', content) oldest_supported_abi_version = match_against( '^OLDEST_SUPPORTED_ABI_VERSION (\d+)$', content) if channel and major and minor and prerelease and prerelease_patch and \ abi_version and oldest_supported_abi_version: return Version(channel, major, minor, patch, prerelease, prerelease_patch, abi_version, oldest_supported_abi_version) else: print("Warning: VERSION file (%s) has wrong format" % version_file) return None # Our schema for releases and archiving is based on an increasing # sequence of numbers. In the svn world this was simply the revision of a # commit, which would always give us a one to one mapping between the number # and the commit. This was true across branches as well, so a number used # to archive a build was always unique and unambiguous. # In git there is no such global number, so we loosen the requirement a bit. # We only use numbers on the master branch (bleeding edge). On branches # we use the version number instead for archiving purposes. # The number on master is the count of commits on the master branch. def GetArchiveVersion(): version = ReadVersionFile() if not version: raise 'Could not get the archive version, parsing the version file failed' if version.channel in ['be', 'integration']: return GetGitNumber() return GetSemanticSDKVersion() def GetGitRevision(): # When building from tarball use tools/GIT_REVISION git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION') try: with open(git_revision_file) as fd: return fd.read() except: pass p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=IsWindows(), cwd=DART_DIR) output, _ = p.communicate() output = output.strip() # We expect a full git hash if len(output) != 40: print("Warning: could not parse git commit, output was %s" % output) return None return output def GetShortGitHash(): p = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=IsWindows(), cwd=DART_DIR) output, _ = p.communicate() output = output.strip() if p.wait() != 0: return None return output def GetLatestDevTag(): cmd = [ 'git', 'for-each-ref', 'refs/tags/*dev*', '--sort=-taggerdate', "--format=%(refname:lstrip=2)", '--count=1', ] p = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=IsWindows(), cwd=DART_DIR) output, _ = p.communicate() if p.wait() != 0: print( "Warning: Could not get the most recent dev branch tag %s" % output) return None return output.strip() def GetGitTimestamp(): p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cd'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=IsWindows(), cwd=DART_DIR) output, _ = p.communicate() if p.wait() != 0: return None return output # To eliminate clashing with older archived builds on bleeding edge we add # a base number bigger the largest svn revision (this also gives us an easy # way of seeing if an archive comes from git based or svn based commits). GIT_NUMBER_BASE = 100000 def GetGitNumber(): p = subprocess.Popen(['git', 'rev-list', 'HEAD', '--count'], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=IsWindows(), cwd=DART_DIR) output, _ = p.communicate() try: number = int(output) return number + GIT_NUMBER_BASE except: print("Warning: could not parse git count, output was %s" % output) return None def ParseGitInfoOutput(output): """Given a git log, determine the latest corresponding svn revision.""" for line in output.split('\n'): tokens = line.split() if len(tokens) > 0 and tokens[0] == 'git-svn-id:': return tokens[1].split('@')[1] return None def ParseSvnInfoOutput(output): revision_match = re.search('Last Changed Rev: (\d+)', output) if revision_match: return revision_match.group(1) return None def RewritePathSeparator(path, workspace): # Paths in test files are always specified using '/' # as the path separator. Replace with the actual # path separator before use. if '/' in path: split_path = path.split('/') path = os.sep.join(split_path) path = os.path.join(workspace, path) if not os.path.exists(path): raise Exception(path) return path def ParseTestOptions(pattern, source, workspace): match = pattern.search(source) if match: return [ RewritePathSeparator(o, workspace) for o in match.group(1).split(' ') ] else: return None def ParseTestOptionsMultiple(pattern, source, workspace): matches = pattern.findall(source) if matches: result = [] for match in matches: if len(match) > 0: result.append([ RewritePathSeparator(o, workspace) for o in match.split(' ') ]) else: result.append([]) return result else: return None def Daemonize(): """ Create a detached background process (daemon). Returns True for the daemon, False for the parent process. See: http://www.faqs.org/faqs/unix-faq/programmer/faq/ "1.7 How do I get my program to act like a daemon?" """ if os.fork() > 0: return False os.setsid() if os.fork() > 0: exit(0) raise return True def PrintError(string): """Writes and flushes a string to stderr.""" sys.stderr.write(string) sys.stderr.write('\n') def CheckedUnlink(name): """Unlink a file without throwing an exception.""" try: os.unlink(name) except OSError as e: PrintError("os.unlink() " + str(e)) def Main(): print("GuessOS() -> ", GuessOS()) print("GuessArchitecture() -> ", GuessArchitecture()) print("GuessCpus() -> ", GuessCpus()) print("IsWindows() -> ", IsWindows()) print("GuessVisualStudioPath() -> ", GuessVisualStudioPath()) print("GetGitRevision() -> ", GetGitRevision()) print("GetGitTimestamp() -> ", GetGitTimestamp()) print("GetVersionFileContent() -> ", GetVersionFileContent()) print("GetGitNumber() -> ", GetGitNumber()) class Error(Exception): pass class ToolError(Exception): """Deprecated exception, use Error instead.""" def __init__(self, value): self.value = value def __str__(self): return repr(self.value) def IsCrashExitCode(exit_code): if IsWindows(): return 0x80000000 & exit_code else: return exit_code < 0 def DiagnoseExitCode(exit_code, command): if IsCrashExitCode(exit_code): sys.stderr.write('Command: %s\nCRASHED with exit code %d (0x%x)\n' % (' '.join(command), exit_code, exit_code & 0xffffffff)) def Touch(name): with file(name, 'a'): os.utime(name, None) def ExecuteCommand(cmd): """Execute a command in a subprocess.""" print('Executing: ' + ' '.join(cmd)) pipe = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=IsWindows()) output = pipe.communicate() if pipe.returncode != 0: raise Exception('Execution failed: ' + str(output)) return pipe.returncode, output # The checked-in SDKs are documented at # https://github.com/dart-lang/sdk/wiki/The-checked-in-SDK-in-tools def CheckedInSdkPath(): # We don't use the normal macos, linux, win32 directory names here, instead, # we use the names that the download_from_google_storage script uses. osdict = {'Darwin': 'mac', 'Linux': 'linux', 'Windows': 'win'} system = platform.system() try: osname = osdict[system] except KeyError: sys.stderr.write('WARNING: platform "%s" not supported\n' % system) return None tools_dir = os.path.dirname(os.path.realpath(__file__)) return os.path.join(tools_dir, 'sdks', 'dart-sdk') def CheckedInSdkExecutable(): name = 'dart' if IsWindows(): name = 'dart.exe' return os.path.join(CheckedInSdkPath(), 'bin', name) def CheckedInSdkCheckExecutable(): executable = CheckedInSdkExecutable() canary_script = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'canary.dart') try: with open(os.devnull, 'wb') as silent_sink: if 0 == subprocess.call([executable, canary_script], stdout=silent_sink): return True except OSError as e: pass return False def CheckLinuxCoreDumpPattern(fatal=False): core_pattern_file = '/proc/sys/kernel/core_pattern' core_pattern = open(core_pattern_file).read() expected_core_pattern = 'core.%p' if core_pattern.strip() != expected_core_pattern: message = ( 'Invalid core_pattern configuration. ' 'The configuration of core dump handling is *not* correct for ' 'a buildbot. The content of {0} must be "{1}" instead of "{2}".'. format(core_pattern_file, expected_core_pattern, core_pattern)) if fatal: raise Exception(message) else: print(message) return False return True class TempDir(object): def __init__(self, prefix=''): self._temp_dir = None self._prefix = prefix def __enter__(self): self._temp_dir = tempfile.mkdtemp(self._prefix) return self._temp_dir def __exit__(self, *_): shutil.rmtree(self._temp_dir, ignore_errors=True) class ChangedWorkingDirectory(object): def __init__(self, working_directory): self._working_directory = working_directory def __enter__(self): self._old_cwd = os.getcwd() print("Enter directory = ", self._working_directory) os.chdir(self._working_directory) def __exit__(self, *_): print("Enter directory = ", self._old_cwd) os.chdir(self._old_cwd) class UnexpectedCrash(object): def __init__(self, test, pid, *binaries): self.test = test self.pid = pid self.binaries = binaries def __str__(self): return "Crash(%s: %s %s)" % (self.test, self.pid, ', '.join( self.binaries)) class PosixCoreDumpEnabler(object): def __init__(self): self._old_limits = None def __enter__(self): self._old_limits = resource.getrlimit(resource.RLIMIT_CORE) resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) def __exit__(self, *_): resource.setrlimit(resource.RLIMIT_CORE, self._old_limits) class LinuxCoreDumpEnabler(PosixCoreDumpEnabler): def __enter__(self): # Bump core limits to unlimited if core_pattern is correctly configured. if CheckLinuxCoreDumpPattern(fatal=False): super(LinuxCoreDumpEnabler, self).__enter__() def __exit__(self, *args): CheckLinuxCoreDumpPattern(fatal=False) super(LinuxCoreDumpEnabler, self).__exit__(*args) class WindowsCoreDumpEnabler(object): """This enabler assumes that Dart binary was built with Crashpad support. In this case DART_CRASHPAD_CRASHES_DIR environment variable allows to specify the location of Crashpad crashes database. Actual minidumps will be written into reports subfolder of the database. """ CRASHPAD_DB_FOLDER = os.path.join(DART_DIR, r'crashes') DUMPS_FOLDER = os.path.join(CRASHPAD_DB_FOLDER, r'reports') def __init__(self): pass def __enter__(self): print("INFO: Enabling coredump archiving into %s" % (WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER)) os.environ[ 'DART_CRASHPAD_CRASHES_DIR'] = WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER def __exit__(self, *_): del os.environ['DART_CRASHPAD_CRASHES_DIR'] def TryUnlink(file): try: os.unlink(file) except Exception as error: print("ERROR: Failed to remove %s: %s" % (file, error)) class BaseCoreDumpArchiver(object): """This class reads coredumps file written by UnexpectedCrashDumpArchiver into the current working directory and uploads all cores and binaries listed in it into Cloud Storage (see pkg/test_runner/lib/src/test_progress.dart). """ # test.dart will write a line for each unexpected crash into this file. _UNEXPECTED_CRASHES_FILE = "unexpected-crashes" def __init__(self, search_dir, output_directory): self._bucket = 'dart-temp-crash-archive' self._binaries_dir = os.getcwd() self._search_dir = search_dir self._output_directory = output_directory def _safe_cleanup(self): try: return self._cleanup() except Exception as error: print("ERROR: Failure during cleanup: %s" % error) return False def __enter__(self): print("INFO: Core dump archiving is activated") # Cleanup any stale files if self._safe_cleanup(): print("WARNING: Found and removed stale coredumps") def __exit__(self, *_): try: crashes = self._find_unexpected_crashes() if crashes: # If we get a ton of crashes, only archive 10 dumps. archive_crashes = crashes[:10] print('Archiving coredumps for crash (if possible):') for crash in archive_crashes: print('----> %s' % crash) sys.stdout.flush() self._archive(archive_crashes) else: print("INFO: No unexpected crashes recorded") dumps = self._find_all_coredumps() if dumps: print("INFO: However there are %d core dumps found" % len(dumps)) for dump in dumps: print("INFO: -> %s" % dump) print() except Exception as error: print("ERROR: Failed to archive crashes: %s" % error) raise finally: self._safe_cleanup() def _archive(self, crashes): files = set() missing = [] for crash in crashes: files.update(crash.binaries) core = self._find_coredump_file(crash) if core: files.add(core) else: missing.append(crash) if self._output_directory is not None and self._is_shard(): print( "INFO: Moving collected dumps and binaries into output directory\n" "INFO: They will be uploaded to isolate server. Look for \"isolated" " out\" under the failed step on the build page.\n" "INFO: For more information see runtime/docs/infra/coredumps.md" ) self._move(files) else: print( "INFO: Uploading collected dumps and binaries into Cloud Storage\n" "INFO: Use `gsutil.py cp from-url to-path` to download them.\n" "INFO: For more information see runtime/docs/infra/coredumps.md" ) self._upload(files) if missing: self._report_missing_crashes(missing, throw=True) # todo(athom): move the logic to decide where to copy core dumps into the recipes. def _is_shard(self): return 'BUILDBOT_BUILDERNAME' not in os.environ def _report_missing_crashes(self, missing, throw=True): missing_as_string = ', '.join([str(c) for c in missing]) other_files = list(glob.glob(os.path.join(self._search_dir, '*'))) sys.stderr.write( "Could not find crash dumps for '%s' in search directory '%s'.\n" "Existing files which *did not* match the pattern inside the search " "directory are are:\n %s\n" % (missing_as_string, self._search_dir, '\n '.join(other_files))) # TODO: Figure out why windows coredump generation does not work. # See http://dartbug.com/36469 if throw and GuessOS() != 'win32': raise Exception('Missing crash dumps for: %s' % missing_as_string) def _get_file_name(self, file): # Sanitize the name: actual cores follow 'core.%d' pattern, crashed # binaries are copied next to cores and named # 'binary.__'. # This should match the code in testing/dart/test_progress.dart name = os.path.basename(file) (prefix, suffix) = name.split('.', 1) is_binary = prefix == 'binary' if is_binary: (mode, arch, binary_name) = suffix.split('_', 2) name = binary_name return (name, is_binary) def _move(self, files): for file in files: print('+++ Moving %s to output_directory (%s)' % (file, self._output_directory)) (name, is_binary) = self._get_file_name(file) destination = os.path.join(self._output_directory, name) shutil.move(file, destination) if is_binary and os.path.exists(file + '.pdb'): # Also move a PDB file if there is one. pdb = os.path.join(self._output_directory, name + '.pdb') shutil.move(file + '.pdb', pdb) def _tar(self, file): (name, is_binary) = self._get_file_name(file) tarname = '%s.tar.gz' % name # Compress the file. tar = tarfile.open(tarname, mode='w:gz') tar.add(file, arcname=name) if is_binary and os.path.exists(file + '.pdb'): # Also add a PDB file if there is one. tar.add(file + '.pdb', arcname=name + '.pdb') tar.close() return tarname def _upload(self, files): bot_utils = GetBotUtils() gsutil = bot_utils.GSUtil() storage_path = '%s/%s/' % (self._bucket, uuid.uuid4()) gs_prefix = 'gs://%s' % storage_path http_prefix = 'https://storage.cloud.google.com/%s' % storage_path print('\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix)) for file in files: tarname = self._tar(file) # Remove / from absolute path to not have // in gs path. gs_url = '%s%s' % (gs_prefix, tarname) http_url = '%s%s' % (http_prefix, tarname) try: gsutil.upload(tarname, gs_url) print('+++ Uploaded %s (%s)' % (gs_url, http_url)) except Exception as error: print('!!! Failed to upload %s, error: %s' % (tarname, error)) TryUnlink(tarname) print('--- Done ---\n') def _find_all_coredumps(self): """Return coredumps that were recorded (if supported by the platform). This method will be overriden by concrete platform specific implementations. """ return [] def _find_unexpected_crashes(self): """Load coredumps file. Each line has the following format: test-name,pid,binary-file1,binary-file2,... """ try: with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f: return [ UnexpectedCrash(*ln.strip('\n').split(',')) for ln in f.readlines() ] except: return [] def _cleanup(self): found = False if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE): os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) found = True for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')): found = True TryUnlink(binary) return found class PosixCoreDumpArchiver(BaseCoreDumpArchiver): def __init__(self, search_dir, output_directory): super(PosixCoreDumpArchiver, self).__init__(search_dir, output_directory) def _cleanup(self): found = super(PosixCoreDumpArchiver, self)._cleanup() for core in glob.glob(os.path.join(self._search_dir, 'core.*')): found = True TryUnlink(core) return found def _find_coredump_file(self, crash): core_filename = os.path.join(self._search_dir, 'core.%s' % crash.pid) if os.path.exists(core_filename): return core_filename class LinuxCoreDumpArchiver(PosixCoreDumpArchiver): def __init__(self, output_directory): super(LinuxCoreDumpArchiver, self).__init__(os.getcwd(), output_directory) class MacOSCoreDumpArchiver(PosixCoreDumpArchiver): def __init__(self, output_directory): super(MacOSCoreDumpArchiver, self).__init__('/cores', output_directory) class WindowsCoreDumpArchiver(BaseCoreDumpArchiver): def __init__(self, output_directory): super(WindowsCoreDumpArchiver, self).__init__( WindowsCoreDumpEnabler.DUMPS_FOLDER, output_directory) self._dumps_by_pid = None # Find CDB.exe in the win_toolchain that we are using. def _find_cdb(self): win_toolchain_json_path = os.path.join(DART_DIR, 'build', 'win_toolchain.json') if not os.path.exists(win_toolchain_json_path): return None with open(win_toolchain_json_path, "r") as f: win_toolchain_info = json.loads(f.read()) win_sdk_path = win_toolchain_info['win_sdk'] # We assume that we are running on 64-bit Windows. # Note: x64 CDB can work with both X64 and IA32 dumps. cdb_path = os.path.join(win_sdk_path, 'Debuggers', 'x64', 'cdb.exe') if not os.path.exists(cdb_path): return None return cdb_path CDBG_PROMPT_RE = re.compile(r'^\d+:\d+>') def _dump_all_stacks(self): # On Windows due to crashpad integration crashes do not produce any # stacktraces. Dump stack traces from dumps Crashpad collected using # CDB (if available). cdb_path = self._find_cdb() if cdb_path is None: return dumps = self._find_all_coredumps() if not dumps: return print("### Collected %d crash dumps" % len(dumps)) for dump in dumps: print() print("### Dumping stacks from %s using CDB" % dump) cdb_output = subprocess.check_output( '"%s" -z "%s" -kqm -c "!uniqstack -b -v -p;qd"' % (cdb_path, dump), stderr=subprocess.STDOUT) # Extract output of uniqstack from the whole output of CDB. output = False for line in cdb_output.split('\n'): if re.match(WindowsCoreDumpArchiver.CDBG_PROMPT_RE, line): output = True elif line.startswith("quit:"): break elif output: print(line) print() print("#############################################") print() def __exit__(self, *args): try: self._dump_all_stacks() except Exception as error: print("ERROR: Unable to dump stacks from dumps: %s" % error) super(WindowsCoreDumpArchiver, self).__exit__(*args) def _cleanup(self): found = super(WindowsCoreDumpArchiver, self)._cleanup() for core in glob.glob(os.path.join(self._search_dir, '*')): found = True TryUnlink(core) return found def _find_all_coredumps(self): pattern = os.path.join(self._search_dir, '*.dmp') return [core_filename for core_filename in glob.glob(pattern)] def _find_coredump_file(self, crash): if self._dumps_by_pid is None: # If this function is invoked the first time then look through the directory # that contains crashes for all dump files and collect pid -> filename # mapping. self._dumps_by_pid = {} minidump = GetMinidumpUtils() pattern = os.path.join(self._search_dir, '*.dmp') for core_filename in glob.glob(pattern): pid = minidump.GetProcessIdFromDump(core_filename) if pid != -1: self._dumps_by_pid[str(pid)] = core_filename if crash.pid in self._dumps_by_pid: return self._dumps_by_pid[crash.pid] def _report_missing_crashes(self, missing, throw=True): # Let's only print the debugging information and not throw. We'll do more # validation for werfault.exe and throw afterwards. super(WindowsCoreDumpArchiver, self)._report_missing_crashes( missing, throw=False) if throw: missing_as_string = ', '.join([str(c) for c in missing]) raise Exception('Missing crash dumps for: %s' % missing_as_string) class IncreasedNumberOfFileDescriptors(object): def __init__(self, nofiles): self._old_limits = None self._limits = (nofiles, nofiles) def __enter__(self): self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE) resource.setrlimit(resource.RLIMIT_NOFILE, self._limits) def __exit__(self, *_): resource.setrlimit(resource.RLIMIT_CORE, self._old_limits) @contextlib.contextmanager def NooptContextManager(): yield def CoreDumpArchiver(args): enabled = '--copy-coredumps' in args prefix = '--output-directory=' output_directory = next( (arg[len(prefix):] for arg in args if arg.startswith(prefix)), None) if not enabled: return NooptContextManager() osname = GuessOS() if osname == 'linux': return contextlib.nested(LinuxCoreDumpEnabler(), LinuxCoreDumpArchiver(output_directory)) elif osname == 'macos': return contextlib.nested(PosixCoreDumpEnabler(), MacOSCoreDumpArchiver(output_directory)) elif osname == 'win32': return contextlib.nested(WindowsCoreDumpEnabler(), WindowsCoreDumpArchiver(output_directory)) else: # We don't have support for MacOS yet. return NooptContextManager() def FileDescriptorLimitIncreaser(): osname = GuessOS() if osname == 'macos': return IncreasedNumberOfFileDescriptors(nofiles=10000) else: assert osname in ('linux', 'win32') # We don't have support for MacOS yet. return NooptContextManager() if __name__ == "__main__": import sys Main()