dart-sdk/tools/utils.py

# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.

# This file contains a set of utilities functions used by other Python-based
# scripts.

import contextlib
import datetime
import glob
import imp
import json
import os
import platform
import re
import shutil
import subprocess
import sys
import tarfile
import tempfile
import uuid

try:
    # Not available on Windows.
    import resource
except:
    pass

DART_DIR = os.path.abspath(os.path.normpath(os.path.join(__file__, '..', '..')))


def GetBotUtils():
    '''Dynamically load the tools/bots/bot_utils.py python module.'''
    return imp.load_source(
        'bot_utils', os.path.join(DART_DIR, 'tools', 'bots', 'bot_utils.py'))


def GetMinidumpUtils():
    '''Dynamically load the tools/minidump.py python module.'''
    return imp.load_source('minidump',
                           os.path.join(DART_DIR, 'tools', 'minidump.py'))


class Version(object):

    def __init__(self, channel, major, minor, patch, prerelease,
                 prerelease_patch, abi_version, oldest_supported_abi_version):
        self.channel = channel
        self.major = major
        self.minor = minor
        self.patch = patch
        self.prerelease = prerelease
        self.prerelease_patch = prerelease_patch
        self.abi_version = abi_version
        self.oldest_supported_abi_version = oldest_supported_abi_version


# Try to guess the host operating system.
def GuessOS():
    os_id = platform.system()
    if os_id == "Linux":
        return "linux"
    elif os_id == "Darwin":
        return "macos"
    elif os_id == "Windows" or os_id == "Microsoft":
        # On Windows Vista platform.system() can return "Microsoft" with some
        # versions of Python, see http://bugs.python.org/issue1082 for details.
        return "win32"
    elif os_id == 'FreeBSD':
        return 'freebsd'
    elif os_id == 'OpenBSD':
        return 'openbsd'
    elif os_id == 'SunOS':
        return 'solaris'
    else:
        return None


# Try to guess the host architecture.
def GuessArchitecture():
    os_id = platform.machine()
    if os_id.startswith('armv6'):
        return 'armv6'
    elif os_id.startswith('arm'):
        return 'arm'
    elif os_id.startswith('aarch64'):
        return 'arm64'
    elif '64' in os_id:
        return 'x64'
    elif (not os_id) or (not re.match('(x|i[3-6])86', os_id) is None):
        return 'ia32'
    elif os_id == 'i86pc':
        return 'ia32'
    else:
        guess_os = GuessOS()
        print ("Warning: Guessing architecture %s based on os %s\n"\
              % (os_id, guess_os))
        if guess_os == 'win32':
            return 'ia32'
        return None


# Try to guess the number of cpus on this machine.
def GuessCpus():
    if os.getenv("DART_NUMBER_OF_CORES") is not None:
        return int(os.getenv("DART_NUMBER_OF_CORES"))
    if os.path.exists("/proc/cpuinfo"):
        return int(
            subprocess.check_output(
                "grep -E '^processor' /proc/cpuinfo | wc -l", shell=True))
    if os.path.exists("/usr/bin/hostinfo"):
        return int(
            subprocess.check_output(
                '/usr/bin/hostinfo |'
                ' grep "processors are logically available." |'
                ' awk "{ print \$1 }"',
                shell=True))
    win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS")
    if win_cpu_count:
        return int(win_cpu_count)
    return 2


def GetWindowsRegistryKeyName(name):
    import win32process
    # Check if python process is 64-bit or if it's 32-bit running in 64-bit OS.
    # We need to know whether host is 64-bit so that we are looking in right
    # registry for Visual Studio path.
    if sys.maxsize > 2**32 or win32process.IsWow64Process():
        wow6432Node = 'Wow6432Node\\'
    else:
        wow6432Node = ''
    return r'SOFTWARE\%s%s' % (wow6432Node, name)


# Try to guess Visual Studio location when buiding on Windows.
def GuessVisualStudioPath():
    defaultPath = r"C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7" \
                  r"\IDE"
    defaultExecutable = "devenv.com"

    if not IsWindows():
        return defaultPath, defaultExecutable

    keyNamesAndExecutables = [
        # Pair for non-Express editions.
        (GetWindowsRegistryKeyName(r'Microsoft\VisualStudio'), 'devenv.com'),
        # Pair for 2012 Express edition.
        (GetWindowsRegistryKeyName(r'Microsoft\VSWinExpress'),
         'VSWinExpress.exe'),
        # Pair for pre-2012 Express editions.
        (GetWindowsRegistryKeyName(r'Microsoft\VCExpress'), 'VCExpress.exe')
    ]

    bestGuess = (0.0, (defaultPath, defaultExecutable))

    import _winreg
    for (keyName, executable) in keyNamesAndExecutables:
        try:
            key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, keyName)
        except WindowsError:
            # Can't find this key - moving on the next one.
            continue

        try:
            subkeyCounter = 0
            while True:
                try:
                    subkeyName = _winreg.EnumKey(key, subkeyCounter)
                    subkeyCounter += 1
                except WindowsError:
                    # Reached end of enumeration. Moving on the next key.
                    break

                match = re.match(r'^\d+\.\d+$', subkeyName)
                if match:
                    with _winreg.OpenKey(key, subkeyName) as subkey:
                        try:
                            (installDir, registrytype) = _winreg.QueryValueEx(
                                subkey, 'InstallDir')
                        except WindowsError:
                            # Can't find value under the key - continue to the next key.
                            continue
                        isExpress = executable != 'devenv.com'
                        if not isExpress and subkeyName == '14.0':
                            # Stop search since if we found non-Express VS2015 version
                            # installed, which is preferred version.
                            return installDir, executable
                        else:
                            version = float(subkeyName)
                            # We prefer higher version of Visual Studio and given equal
                            # version numbers we prefer non-Express edition.
                            if version > bestGuess[0]:
                                bestGuess = (version, (installDir, executable))
        finally:
            _winreg.CloseKey(key)
    return bestGuess[1]


# Returns true if we're running under Windows.
def IsWindows():
    return GuessOS() == 'win32'


# Reads a text file into an array of strings - one for each
# line. Strips comments in the process.
def ReadLinesFrom(name):
    result = []
    for line in open(name):
        if '#' in line:
            line = line[:line.find('#')]
        line = line.strip()
        if len(line) == 0:
            continue
        result.append(line)
    return result


# Filters out all arguments until the next '--' argument
# occurs.
def ListArgCallback(option, value, parser):
    if value is None:
        value = []

    for arg in parser.rargs:
        if arg[:2].startswith('--'):
            break
        value.append(arg)

    del parser.rargs[:len(value)]
    setattr(parser.values, option.dest, value)


# Filters out all argument until the first non '-' or the
# '--' argument occurs.
def ListDartArgCallback(option, value, parser):
    if value is None:
        value = []

    for arg in parser.rargs:
        if arg[:2].startswith('--') or arg[0] != '-':
            break
        value.append(arg)

    del parser.rargs[:len(value)]
    setattr(parser.values, option.dest, value)


# Mapping table between build mode and build configuration.
BUILD_MODES = {
    'debug': 'Debug',
    'release': 'Release',
    'product': 'Product',
}

# Mapping table between build mode and build configuration.
BUILD_SANITIZERS = {
    None: '',
    'none': '',
    'asan': 'ASAN',
    'lsan': 'LSAN',
    'msan': 'MSAN',
    'tsan': 'TSAN',
    'ubsan': 'UBSAN',
}

# Mapping table between OS and build output location.
BUILD_ROOT = {
    'win32': os.path.join('out'),
    'linux': os.path.join('out'),
    'freebsd': os.path.join('out'),
    'macos': os.path.join('xcodebuild'),
}

# Note: gn expects these to be lower case.
ARCH_FAMILY = {
    'ia32': 'ia32',
    'x64': 'ia32',
    'arm': 'arm',
    'armv6': 'arm',
    'arm64': 'arm',
    'arm_x64': 'arm',
    'simarm': 'ia32',
    'simarmv6': 'ia32',
    'simarm64': 'ia32',
    'simarm_x64': 'ia32',
}

ARCH_GUESS = GuessArchitecture()
BASE_DIR = os.path.abspath(os.path.join(os.curdir, '..'))
DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..'))
VERSION_FILE = os.path.join(DART_DIR, 'tools', 'VERSION')


def GetBuildbotGSUtilPath():
    gsutil = '/b/build/scripts/slave/gsutil'
    if platform.system() == 'Windows':
        gsutil = 'e:\\\\b\\build\\scripts\\slave\\gsutil'
    return gsutil


def GetBuildMode(mode):
    return BUILD_MODES[mode]


def GetBuildSanitizer(sanitizer):
    return BUILD_SANITIZERS[sanitizer]


def GetArchFamily(arch):
    return ARCH_FAMILY[arch]


def IsCrossBuild(target_os, arch):
    host_arch = ARCH_GUESS
    return ((GetArchFamily(host_arch) != GetArchFamily(arch)) or
            (target_os != GuessOS()))

# TODO(38701): Remove use_nnbd once the forked NNBD SDK is merged back in.
def GetBuildConf(mode, arch, conf_os=None, sanitizer=None, use_nnbd=False):
    nnbd = "NNBD" if use_nnbd else ""
    if conf_os == 'android':
        return '%s%s%s%s' % (GetBuildMode(mode), conf_os.title(), arch.upper(),
                             nnbd)
    else:
        # Ask for a cross build if the host and target architectures don't match.
        host_arch = ARCH_GUESS
        cross_build = ''
        if GetArchFamily(host_arch) != GetArchFamily(arch):
            cross_build = 'X'
        return '%s%s%s%s%s' % (GetBuildMode(mode), GetBuildSanitizer(sanitizer),
                               cross_build, arch.upper(), nnbd)


def GetBuildDir(host_os):
    return BUILD_ROOT[host_os]


# TODO(38701): Remove use_nnbd once the forked NNBD SDK is merged back in.
def GetBuildRoot(host_os,
                 mode=None,
                 arch=None,
                 target_os=None,
                 sanitizer=None,
                 use_nnbd=False):
    build_root = GetBuildDir(host_os)
    if mode:
        build_root = os.path.join(
            build_root, GetBuildConf(mode, arch, target_os, sanitizer,
                                     use_nnbd))
    return build_root


# TODO(38701): Remove use_nnbd once the forked NNBD SDK is merged back in.
def GetBuildSdkBin(host_os,
                   mode=None,
                   arch=None,
                   target_os=None,
                   use_nnbd=False):
    build_root = GetBuildRoot(host_os, mode, arch, target_os, use_nnbd)
    return os.path.join(build_root, 'dart-sdk', 'bin')


def GetBaseDir():
    return BASE_DIR


def GetShortVersion():
    version = ReadVersionFile()
    return ('%s.%s.%s.%s.%s' % (version.major, version.minor, version.patch,
                                version.prerelease, version.prerelease_patch))


def GetSemanticSDKVersion(no_git_hash=False, version_file=None):
    version = ReadVersionFile(version_file)
    if not version:
        return None

    if version.channel == 'be':
        postfix = '-edge' if no_git_hash else '-edge.%s' % GetGitRevision()
    elif version.channel == 'dev':
        postfix = '-dev.%s.%s' % (version.prerelease, version.prerelease_patch)
    else:
        assert version.channel == 'stable'
        postfix = ''

    return '%s.%s.%s%s' % (version.major, version.minor, version.patch, postfix)


def GetVersion(no_git_hash=False, version_file=None):
    return GetSemanticSDKVersion(no_git_hash, version_file)


# The editor used to produce the VERSION file put on gcs. We now produce this
# in the bots archiving the sdk.
# The content looks like this:
#{
#  "date": "2015-05-28",
#  "version": "1.11.0-edge.131653",
#  "revision": "535394c2657ede445142d8a92486d3899bbf49b5"
#}
def GetVersionFileContent():
    result = {
        "date": str(datetime.date.today()),
        "version": GetVersion(),
        "revision": GetGitRevision()
    }
    return json.dumps(result, indent=2)


def GetChannel():
    version = ReadVersionFile()
    return version.channel


def GetUserName():
    key = 'USER'
    if sys.platform == 'win32':
        key = 'USERNAME'
    return os.environ.get(key, '')


def GetAbiVersion(version_file=None):
    version = ReadVersionFile(version_file)
    return version.abi_version


def GetOldestSupportedAbiVersion(version_file=None):
    version = ReadVersionFile(version_file)
    return version.oldest_supported_abi_version


def ReadVersionFile(version_file=None):

    def match_against(pattern, file_content):
        match = re.search(pattern, file_content, flags=re.MULTILINE)
        if match:
            return match.group(1)
        return None

    if version_file == None:
        version_file = VERSION_FILE

    try:
        fd = open(version_file)
        content = fd.read()
        fd.close()
    except:
        print("Warning: Couldn't read VERSION file (%s)" % version_file)
        return None

    channel = match_against('^CHANNEL ([A-Za-z0-9]+)$', content)
    major = match_against('^MAJOR (\d+)$', content)
    minor = match_against('^MINOR (\d+)$', content)
    patch = match_against('^PATCH (\d+)$', content)
    prerelease = match_against('^PRERELEASE (\d+)$', content)
    prerelease_patch = match_against('^PRERELEASE_PATCH (\d+)$', content)
    abi_version = match_against('^ABI_VERSION (\d+)$', content)
    oldest_supported_abi_version = match_against(
        '^OLDEST_SUPPORTED_ABI_VERSION (\d+)$', content)

    if channel and major and minor and prerelease and prerelease_patch and \
        abi_version and oldest_supported_abi_version:
        return Version(channel, major, minor, patch, prerelease,
                       prerelease_patch, abi_version,
                       oldest_supported_abi_version)
    else:
        print("Warning: VERSION file (%s) has wrong format" % version_file)
        return None


# Our schema for releases and archiving is based on an increasing
# sequence of numbers. In the svn world this was simply the revision of a
# commit, which would always give us a one to one mapping between the number
# and the commit. This was true across branches as well, so a number used
# to archive a build was always unique and unambiguous.
# In git there is no such global number, so we loosen the requirement a bit.
# We only use numbers on the master branch (bleeding edge). On branches
# we use the version number instead for archiving purposes.
# The number on master is the count of commits on the master branch.
def GetArchiveVersion():
    version = ReadVersionFile()
    if not version:
        raise 'Could not get the archive version, parsing the version file failed'
    if version.channel in ['be', 'integration']:
        return GetGitNumber()
    return GetSemanticSDKVersion()


def GetGitRevision():
    # When building from tarball use tools/GIT_REVISION
    git_revision_file = os.path.join(DART_DIR, 'tools', 'GIT_REVISION')
    try:
        with open(git_revision_file) as fd:
            return fd.read()
    except:
        pass
    p = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
                         shell=IsWindows(),
                         cwd=DART_DIR)
    output, _ = p.communicate()
    output = output.strip()
    # We expect a full git hash
    if len(output) != 40:
        print("Warning: could not parse git commit, output was %s" % output)
        return None
    return output


def GetShortGitHash():
    p = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
                         shell=IsWindows(),
                         cwd=DART_DIR)
    output, _ = p.communicate()
    output = output.strip()
    if p.wait() != 0:
        return None
    return output


def GetLatestDevTag():
    cmd = [
        'git',
        'for-each-ref',
        'refs/tags/*dev*',
        '--sort=-taggerdate',
        "--format=%(refname:lstrip=2)",
        '--count=1',
    ]
    p = subprocess.Popen(
        cmd,
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT,
        shell=IsWindows(),
        cwd=DART_DIR)
    output, _ = p.communicate()
    if p.wait() != 0:
        print(
            "Warning: Could not get the most recent dev branch tag %s" % output)
        return None
    return output.strip()


def GetGitTimestamp():
    p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cd'],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
                         shell=IsWindows(),
                         cwd=DART_DIR)
    output, _ = p.communicate()
    if p.wait() != 0:
        return None
    return output


# To eliminate clashing with older archived builds on bleeding edge we add
# a base number bigger the largest svn revision (this also gives us an easy
# way of seeing if an archive comes from git based or svn based commits).
GIT_NUMBER_BASE = 100000


def GetGitNumber():
    p = subprocess.Popen(['git', 'rev-list', 'HEAD', '--count'],
                         stdout=subprocess.PIPE,
                         stderr=subprocess.STDOUT,
                         shell=IsWindows(),
                         cwd=DART_DIR)
    output, _ = p.communicate()
    try:
        number = int(output)
        return number + GIT_NUMBER_BASE
    except:
        print("Warning: could not parse git count, output was %s" % output)
    return None


def ParseGitInfoOutput(output):
    """Given a git log, determine the latest corresponding svn revision."""
    for line in output.split('\n'):
        tokens = line.split()
        if len(tokens) > 0 and tokens[0] == 'git-svn-id:':
            return tokens[1].split('@')[1]
    return None


def ParseSvnInfoOutput(output):
    revision_match = re.search('Last Changed Rev: (\d+)', output)
    if revision_match:
        return revision_match.group(1)
    return None


def RewritePathSeparator(path, workspace):
    # Paths in test files are always specified using '/'
    # as the path separator. Replace with the actual
    # path separator before use.
    if '/' in path:
        split_path = path.split('/')
        path = os.sep.join(split_path)
        path = os.path.join(workspace, path)
        if not os.path.exists(path):
            raise Exception(path)
    return path


def ParseTestOptions(pattern, source, workspace):
    match = pattern.search(source)
    if match:
        return [
            RewritePathSeparator(o, workspace)
            for o in match.group(1).split(' ')
        ]
    else:
        return None


def ParseTestOptionsMultiple(pattern, source, workspace):
    matches = pattern.findall(source)
    if matches:
        result = []
        for match in matches:
            if len(match) > 0:
                result.append([
                    RewritePathSeparator(o, workspace) for o in match.split(' ')
                ])
            else:
                result.append([])
        return result
    else:
        return None


def Daemonize():
    """
  Create a detached background process (daemon). Returns True for
  the daemon, False for the parent process.
  See: http://www.faqs.org/faqs/unix-faq/programmer/faq/
  "1.7 How do I get my program to act like a daemon?"
  """
    if os.fork() > 0:
        return False
    os.setsid()
    if os.fork() > 0:
        exit(0)
        raise
    return True


def PrintError(string):
    """Writes and flushes a string to stderr."""
    sys.stderr.write(string)
    sys.stderr.write('\n')


def CheckedUnlink(name):
    """Unlink a file without throwing an exception."""
    try:
        os.unlink(name)
    except OSError as e:
        PrintError("os.unlink() " + str(e))


def Main():
    print("GuessOS() -> ", GuessOS())
    print("GuessArchitecture() -> ", GuessArchitecture())
    print("GuessCpus() -> ", GuessCpus())
    print("IsWindows() -> ", IsWindows())
    print("GuessVisualStudioPath() -> ", GuessVisualStudioPath())
    print("GetGitRevision() -> ", GetGitRevision())
    print("GetGitTimestamp() -> ", GetGitTimestamp())
    print("GetVersionFileContent() -> ", GetVersionFileContent())
    print("GetGitNumber() -> ", GetGitNumber())


class Error(Exception):
    pass


class ToolError(Exception):
    """Deprecated exception, use Error instead."""

    def __init__(self, value):
        self.value = value

    def __str__(self):
        return repr(self.value)


def IsCrashExitCode(exit_code):
    if IsWindows():
        return 0x80000000 & exit_code
    else:
        return exit_code < 0


def DiagnoseExitCode(exit_code, command):
    if IsCrashExitCode(exit_code):
        sys.stderr.write('Command: %s\nCRASHED with exit code %d (0x%x)\n' %
                         (' '.join(command), exit_code, exit_code & 0xffffffff))


def Touch(name):
    with file(name, 'a'):
        os.utime(name, None)


def ExecuteCommand(cmd):
    """Execute a command in a subprocess."""
    print('Executing: ' + ' '.join(cmd))
    pipe = subprocess.Popen(
        cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=IsWindows())
    output = pipe.communicate()
    if pipe.returncode != 0:
        raise Exception('Execution failed: ' + str(output))
    return pipe.returncode, output


# The checked-in SDKs are documented at
#     https://github.com/dart-lang/sdk/wiki/The-checked-in-SDK-in-tools
def CheckedInSdkPath():
    # We don't use the normal macos, linux, win32 directory names here, instead,
    # we use the names that the download_from_google_storage script uses.
    osdict = {'Darwin': 'mac', 'Linux': 'linux', 'Windows': 'win'}
    system = platform.system()
    try:
        osname = osdict[system]
    except KeyError:
        sys.stderr.write('WARNING: platform "%s" not supported\n' % system)
        return None
    tools_dir = os.path.dirname(os.path.realpath(__file__))
    return os.path.join(tools_dir, 'sdks', 'dart-sdk')


def CheckedInSdkExecutable():
    name = 'dart'
    if IsWindows():
        name = 'dart.exe'
    return os.path.join(CheckedInSdkPath(), 'bin', name)


def CheckedInSdkCheckExecutable():
    executable = CheckedInSdkExecutable()
    canary_script = os.path.join(
        os.path.dirname(os.path.realpath(__file__)), 'canary.dart')
    try:
        with open(os.devnull, 'wb') as silent_sink:
            if 0 == subprocess.call([executable, canary_script],
                                    stdout=silent_sink):
                return True
    except OSError as e:
        pass
    return False


def CheckLinuxCoreDumpPattern(fatal=False):
    core_pattern_file = '/proc/sys/kernel/core_pattern'
    core_pattern = open(core_pattern_file).read()

    expected_core_pattern = 'core.%p'
    if core_pattern.strip() != expected_core_pattern:
        message = (
            'Invalid core_pattern configuration. '
            'The configuration of core dump handling is *not* correct for '
            'a buildbot. The content of {0} must be "{1}" instead of "{2}".'.
            format(core_pattern_file, expected_core_pattern, core_pattern))
        if fatal:
            raise Exception(message)
        else:
            print(message)
            return False
    return True


class TempDir(object):

    def __init__(self, prefix=''):
        self._temp_dir = None
        self._prefix = prefix

    def __enter__(self):
        self._temp_dir = tempfile.mkdtemp(self._prefix)
        return self._temp_dir

    def __exit__(self, *_):
        shutil.rmtree(self._temp_dir, ignore_errors=True)


class ChangedWorkingDirectory(object):

    def __init__(self, working_directory):
        self._working_directory = working_directory

    def __enter__(self):
        self._old_cwd = os.getcwd()
        print("Enter directory = ", self._working_directory)
        os.chdir(self._working_directory)

    def __exit__(self, *_):
        print("Enter directory = ", self._old_cwd)
        os.chdir(self._old_cwd)


class UnexpectedCrash(object):

    def __init__(self, test, pid, *binaries):
        self.test = test
        self.pid = pid
        self.binaries = binaries

    def __str__(self):
        return "Crash(%s: %s %s)" % (self.test, self.pid, ', '.join(
            self.binaries))


class PosixCoreDumpEnabler(object):

    def __init__(self):
        self._old_limits = None

    def __enter__(self):
        self._old_limits = resource.getrlimit(resource.RLIMIT_CORE)
        resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))

    def __exit__(self, *_):
        if self._old_limits != None:
            resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)


class LinuxCoreDumpEnabler(PosixCoreDumpEnabler):

    def __enter__(self):
        # Bump core limits to unlimited if core_pattern is correctly configured.
        if CheckLinuxCoreDumpPattern(fatal=False):
            super(LinuxCoreDumpEnabler, self).__enter__()

    def __exit__(self, *args):
        CheckLinuxCoreDumpPattern(fatal=False)
        super(LinuxCoreDumpEnabler, self).__exit__(*args)


class WindowsCoreDumpEnabler(object):
    """This enabler assumes that Dart binary was built with Crashpad support.
  In this case DART_CRASHPAD_CRASHES_DIR environment variable allows to
  specify the location of Crashpad crashes database. Actual minidumps will
  be written into reports subfolder of the database.
  """
    CRASHPAD_DB_FOLDER = os.path.join(DART_DIR, r'crashes')
    DUMPS_FOLDER = os.path.join(CRASHPAD_DB_FOLDER, r'reports')

    def __init__(self):
        pass

    def __enter__(self):
        print("INFO: Enabling coredump archiving into %s" %
              (WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER))
        os.environ[
            'DART_CRASHPAD_CRASHES_DIR'] = WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER

    def __exit__(self, *_):
        del os.environ['DART_CRASHPAD_CRASHES_DIR']


def TryUnlink(file):
    try:
        os.unlink(file)
    except Exception as error:
        print("ERROR: Failed to remove %s: %s" % (file, error))


class BaseCoreDumpArchiver(object):
    """This class reads coredumps file written by UnexpectedCrashDumpArchiver
  into the current working directory and uploads all cores and binaries
  listed in it into Cloud Storage (see
  pkg/test_runner/lib/src/test_progress.dart).
  """

    # test.dart will write a line for each unexpected crash into this file.
    _UNEXPECTED_CRASHES_FILE = "unexpected-crashes"

    def __init__(self, search_dir, output_directory):
        self._bucket = 'dart-temp-crash-archive'
        self._binaries_dir = os.getcwd()
        self._search_dir = search_dir
        self._output_directory = output_directory

    def _safe_cleanup(self):
        try:
            return self._cleanup()
        except Exception as error:
            print("ERROR: Failure during cleanup: %s" % error)
            return False

    def __enter__(self):
        print("INFO: Core dump archiving is activated")

        # Cleanup any stale files
        if self._safe_cleanup():
            print("WARNING: Found and removed stale coredumps")

    def __exit__(self, *_):
        try:
            crashes = self._find_unexpected_crashes()
            if crashes:
                # If we get a ton of crashes, only archive 10 dumps.
                archive_crashes = crashes[:10]
                print('Archiving coredumps for crash (if possible):')
                for crash in archive_crashes:
                    print('----> %s' % crash)

                sys.stdout.flush()

                self._archive(archive_crashes)
            else:
                print("INFO: No unexpected crashes recorded")
                dumps = self._find_all_coredumps()
                if dumps:
                    print("INFO: However there are %d core dumps found" %
                          len(dumps))
                    for dump in dumps:
                        print("INFO:        -> %s" % dump)
                    print()
        except Exception as error:
            print("ERROR: Failed to archive crashes: %s" % error)
            raise

        finally:
            self._safe_cleanup()

    def _archive(self, crashes):
        files = set()
        missing = []
        for crash in crashes:
            files.update(crash.binaries)
            core = self._find_coredump_file(crash)
            if core:
                files.add(core)
            else:
                missing.append(crash)
        if self._output_directory is not None and self._is_shard():
            print(
                "INFO: Moving collected dumps and binaries into output directory\n"
                "INFO: They will be uploaded to isolate server. Look for \"isolated"
                " out\" under the failed step on the build page.\n"
                "INFO: For more information see runtime/docs/infra/coredumps.md"
            )
            self._move(files)
        else:
            print(
                "INFO: Uploading collected dumps and binaries into Cloud Storage\n"
                "INFO: Use `gsutil.py cp from-url to-path` to download them.\n"
                "INFO: For more information see runtime/docs/infra/coredumps.md"
            )
            self._upload(files)

        if missing:
            self._report_missing_crashes(missing, throw=True)

    # todo(athom): move the logic to decide where to copy core dumps into the recipes.
    def _is_shard(self):
        return 'BUILDBOT_BUILDERNAME' not in os.environ

    def _report_missing_crashes(self, missing, throw=True):
        missing_as_string = ', '.join([str(c) for c in missing])
        other_files = list(glob.glob(os.path.join(self._search_dir, '*')))
        sys.stderr.write(
            "Could not find crash dumps for '%s' in search directory '%s'.\n"
            "Existing files which *did not* match the pattern inside the search "
            "directory are are:\n  %s\n" % (missing_as_string, self._search_dir,
                                            '\n  '.join(other_files)))
        # TODO: Figure out why windows coredump generation does not work.
        # See http://dartbug.com/36469
        if throw and GuessOS() != 'win32':
            raise Exception('Missing crash dumps for: %s' % missing_as_string)

    def _get_file_name(self, file):
        # Sanitize the name: actual cores follow 'core.%d' pattern, crashed
        # binaries are copied next to cores and named
        # 'binary.<mode>_<arch>_<binary_name>'.
        # This should match the code in testing/dart/test_progress.dart
        name = os.path.basename(file)
        (prefix, suffix) = name.split('.', 1)
        is_binary = prefix == 'binary'
        if is_binary:
            (mode, arch, binary_name) = suffix.split('_', 2)
            name = binary_name
        return (name, is_binary)

    def _move(self, files):
        for file in files:
            print('+++ Moving %s to output_directory (%s)' %
                  (file, self._output_directory))
            (name, is_binary) = self._get_file_name(file)
            destination = os.path.join(self._output_directory, name)
            shutil.move(file, destination)
            if is_binary and os.path.exists(file + '.pdb'):
                # Also move a PDB file if there is one.
                pdb = os.path.join(self._output_directory, name + '.pdb')
                shutil.move(file + '.pdb', pdb)

    def _tar(self, file):
        (name, is_binary) = self._get_file_name(file)
        tarname = '%s.tar.gz' % name

        # Compress the file.
        tar = tarfile.open(tarname, mode='w:gz')
        tar.add(file, arcname=name)
        if is_binary and os.path.exists(file + '.pdb'):
            # Also add a PDB file if there is one.
            tar.add(file + '.pdb', arcname=name + '.pdb')
        tar.close()
        return tarname

    def _upload(self, files):
        bot_utils = GetBotUtils()
        gsutil = bot_utils.GSUtil()
        storage_path = '%s/%s/' % (self._bucket, uuid.uuid4())
        gs_prefix = 'gs://%s' % storage_path
        http_prefix = 'https://storage.cloud.google.com/%s' % storage_path

        print('\n--- Uploading into %s (%s) ---' % (gs_prefix, http_prefix))
        for file in files:
            tarname = self._tar(file)

            # Remove / from absolute path to not have // in gs path.
            gs_url = '%s%s' % (gs_prefix, tarname)
            http_url = '%s%s' % (http_prefix, tarname)

            try:
                gsutil.upload(tarname, gs_url)
                print('+++ Uploaded %s (%s)' % (gs_url, http_url))
            except Exception as error:
                print('!!! Failed to upload %s, error: %s' % (tarname, error))

            TryUnlink(tarname)

        print('--- Done ---\n')

    def _find_all_coredumps(self):
        """Return coredumps that were recorded (if supported by the platform).
    This method will be overriden by concrete platform specific implementations.
    """
        return []

    def _find_unexpected_crashes(self):
        """Load coredumps file. Each line has the following format:

        test-name,pid,binary-file1,binary-file2,...
    """
        try:
            with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f:
                return [
                    UnexpectedCrash(*ln.strip('\n').split(','))
                    for ln in f.readlines()
                ]
        except:
            return []

    def _cleanup(self):
        found = False
        if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE):
            os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE)
            found = True
        for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')):
            found = True
            TryUnlink(binary)

        return found


class PosixCoreDumpArchiver(BaseCoreDumpArchiver):

    def __init__(self, search_dir, output_directory):
        super(PosixCoreDumpArchiver, self).__init__(search_dir,
                                                    output_directory)

    def _cleanup(self):
        found = super(PosixCoreDumpArchiver, self)._cleanup()
        for core in glob.glob(os.path.join(self._search_dir, 'core.*')):
            found = True
            TryUnlink(core)
        return found

    def _find_coredump_file(self, crash):
        core_filename = os.path.join(self._search_dir, 'core.%s' % crash.pid)
        if os.path.exists(core_filename):
            return core_filename


class LinuxCoreDumpArchiver(PosixCoreDumpArchiver):

    def __init__(self, output_directory):
        super(LinuxCoreDumpArchiver, self).__init__(os.getcwd(),
                                                    output_directory)


class MacOSCoreDumpArchiver(PosixCoreDumpArchiver):

    def __init__(self, output_directory):
        super(MacOSCoreDumpArchiver, self).__init__('/cores', output_directory)


class WindowsCoreDumpArchiver(BaseCoreDumpArchiver):

    def __init__(self, output_directory):
        super(WindowsCoreDumpArchiver, self).__init__(
            WindowsCoreDumpEnabler.DUMPS_FOLDER, output_directory)
        self._dumps_by_pid = None

    # Find CDB.exe in the win_toolchain that we are using.
    def _find_cdb(self):
        win_toolchain_json_path = os.path.join(DART_DIR, 'build',
                                               'win_toolchain.json')
        if not os.path.exists(win_toolchain_json_path):
            return None

        with open(win_toolchain_json_path, "r") as f:
            win_toolchain_info = json.loads(f.read())

        win_sdk_path = win_toolchain_info['win_sdk']

        # We assume that we are running on 64-bit Windows.
        # Note: x64 CDB can work with both X64 and IA32 dumps.
        cdb_path = os.path.join(win_sdk_path, 'Debuggers', 'x64', 'cdb.exe')
        if not os.path.exists(cdb_path):
            return None

        return cdb_path

    CDBG_PROMPT_RE = re.compile(r'^\d+:\d+>')

    def _dump_all_stacks(self):
        # On Windows due to crashpad integration crashes do not produce any
        # stacktraces. Dump stack traces from dumps Crashpad collected using
        # CDB (if available).
        cdb_path = self._find_cdb()
        if cdb_path is None:
            return

        dumps = self._find_all_coredumps()
        if not dumps:
            return

        print("### Collected %d crash dumps" % len(dumps))
        for dump in dumps:
            print()
            print("### Dumping stacks from %s using CDB" % dump)
            cdb_output = subprocess.check_output(
                '"%s" -z "%s" -kqm -c "!uniqstack -b -v -p;qd"' % (cdb_path,
                                                                   dump),
                stderr=subprocess.STDOUT)
            # Extract output of uniqstack from the whole output of CDB.
            output = False
            for line in cdb_output.split('\n'):
                if re.match(WindowsCoreDumpArchiver.CDBG_PROMPT_RE, line):
                    output = True
                elif line.startswith("quit:"):
                    break
                elif output:
                    print(line)
        print()
        print("#############################################")
        print()

    def __exit__(self, *args):
        try:
            self._dump_all_stacks()
        except Exception as error:
            print("ERROR: Unable to dump stacks from dumps: %s" % error)

        super(WindowsCoreDumpArchiver, self).__exit__(*args)

    def _cleanup(self):
        found = super(WindowsCoreDumpArchiver, self)._cleanup()
        for core in glob.glob(os.path.join(self._search_dir, '*')):
            found = True
            TryUnlink(core)
        return found

    def _find_all_coredumps(self):
        pattern = os.path.join(self._search_dir, '*.dmp')
        return [core_filename for core_filename in glob.glob(pattern)]

    def _find_coredump_file(self, crash):
        if self._dumps_by_pid is None:
            # If this function is invoked the first time then look through the directory
            # that contains crashes for all dump files and collect pid -> filename
            # mapping.
            self._dumps_by_pid = {}
            minidump = GetMinidumpUtils()
            pattern = os.path.join(self._search_dir, '*.dmp')
            for core_filename in glob.glob(pattern):
                pid = minidump.GetProcessIdFromDump(core_filename)
                if pid != -1:
                    self._dumps_by_pid[str(pid)] = core_filename
        if crash.pid in self._dumps_by_pid:
            return self._dumps_by_pid[crash.pid]

    def _report_missing_crashes(self, missing, throw=True):
        # Let's only print the debugging information and not throw. We'll do more
        # validation for werfault.exe and throw afterwards.
        super(WindowsCoreDumpArchiver, self)._report_missing_crashes(
            missing, throw=False)

        if throw:
            missing_as_string = ', '.join([str(c) for c in missing])
            raise Exception('Missing crash dumps for: %s' % missing_as_string)


class IncreasedNumberOfFileDescriptors(object):

    def __init__(self, nofiles):
        self._old_limits = None
        self._limits = (nofiles, nofiles)

    def __enter__(self):
        self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE)
        resource.setrlimit(resource.RLIMIT_NOFILE, self._limits)

    def __exit__(self, *_):
        resource.setrlimit(resource.RLIMIT_CORE, self._old_limits)


@contextlib.contextmanager
def NooptContextManager():
    yield


def CoreDumpArchiver(args):
    enabled = '--copy-coredumps' in args
    prefix = '--output-directory='
    output_directory = next(
        (arg[len(prefix):] for arg in args if arg.startswith(prefix)), None)

    if not enabled:
        return NooptContextManager()

    osname = GuessOS()
    if osname == 'linux':
        return contextlib.nested(LinuxCoreDumpEnabler(),
                                 LinuxCoreDumpArchiver(output_directory))
    elif osname == 'macos':
        return contextlib.nested(PosixCoreDumpEnabler(),
                                 MacOSCoreDumpArchiver(output_directory))
    elif osname == 'win32':
        return contextlib.nested(WindowsCoreDumpEnabler(),
                                 WindowsCoreDumpArchiver(output_directory))
    else:
        # We don't have support for MacOS yet.
        return NooptContextManager()


def FileDescriptorLimitIncreaser():
    osname = GuessOS()
    if osname == 'macos':
        return IncreasedNumberOfFileDescriptors(nofiles=10000)
    else:
        assert osname in ('linux', 'win32')
        # We don't have support for MacOS yet.
        return NooptContextManager()


if __name__ == "__main__":
    import sys
    Main()