Remove old annotated_steps buildbot scripts and dispatcher

BUG=
R=kustermann@google.com

Review-Url: https://codereview.chromium.org/2895973002 .
This commit is contained in:
William Hesse 2017-05-23 16:16:59 +02:00
parent 3aad4a9cd8
commit a4adc426e1
12 changed files with 0 additions and 1161 deletions

View file

@ -1,108 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Dart client buildbot steps
Calls a script in tools/bots whose name is based on the name of the bot.
"""
import imp
import os
import re
import socket
import subprocess
import sys
BUILDER_NAME = 'BUILDBOT_BUILDERNAME'
BUILDER_CLOBBER = 'BUILDBOT_CLOBBER'
def GetName():
"""Returns the name of the bot.
"""
name = None
# Populate via builder environment variables.
name = os.environ.get(BUILDER_NAME)
# Fall back if not on builder.
if not name:
name = socket.gethostname().split('.')[0]
return name
def ProcessBot(name, target, custom_env=None):
'''
Build and test the named bot target (compiler, android, pub). We look for
the supporting script in tools/bots/ to run the tests and build.
'''
print 'Process%s' % target.capitalize()
has_shell = False
environment = custom_env or os.environ
if '-win' in name:
# In Windows we need to run in the shell, so that we have all the
# environment variables available.
has_shell = True
return subprocess.call([sys.executable,
os.path.join('tools', 'bots', target + '.py')],
env=environment, shell=has_shell)
def ClobberBuilder():
""" Clobber the builder before we do the build.
"""
cmd = [sys.executable,
'./tools/clean_output_directory.py']
print 'Clobbering %s' % (' '.join(cmd))
return subprocess.call(cmd)
def GetShouldClobber():
return os.environ.get(BUILDER_CLOBBER) == "1"
def main():
if len(sys.argv) == 0:
print 'Script pathname not known, giving up.'
return 1
scriptdir = os.path.dirname(sys.argv[0])
# Get at the top-level directory. This script is in client/tools
os.chdir(os.path.abspath(os.path.join(scriptdir, os.pardir, os.pardir)))
if GetShouldClobber():
print '@@@BUILD_STEP Clobber@@@'
status = ClobberBuilder()
if status != 0:
print '@@@STEP_FAILURE@@@'
return status
name = GetName()
if name.startswith('pkg-'):
status = ProcessBot(name, 'pkg')
elif name.startswith('pub-'):
status = ProcessBot(name, 'pub')
elif name.startswith('vm-android'):
status = ProcessBot(name, 'android')
elif name.startswith('dart-sdk'):
status = ProcessBot(name, 'dart_sdk')
elif name.startswith('cross') or name.startswith('target'):
status = ProcessBot(name, 'cross-vm')
elif name.startswith('linux-distribution-support'):
status = ProcessBot(name, 'linux_distribution_support')
elif name.startswith('version-checker'):
status = ProcessBot(name, 'version_checker')
elif name.startswith('dart2js-dump-info'):
status = ProcessBot(name, 'dart2js_dump_info')
else:
status = ProcessBot(name, 'compiler')
if status:
print '@@@STEP_FAILURE@@@'
return status
if __name__ == '__main__':
sys.exit(main())

View file

@ -1,420 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""
Dart2js buildbot steps
Runs tests for the dart2js compiler.
"""
import os
import platform
import re
import shutil
import socket
import string
import subprocess
import sys
import bot
DARTIUM_BUILDER = r'none-dartium-(linux|mac|windows)'
DART2JS_BUILDER = (
r'dart2js-(linux|mac|windows)(-(jsshell))?-(debug|release)(-(checked|host-checked))?(-(host-checked))?(-(minified))?(-(x64))?(-(cps))?-?(\d*)-?(\d*)')
DART2JS_FULL_BUILDER = r'full-(linux|mac|win7|win8)(-(ie10|ie11))?(-checked)?(-minified)?-(\d+)-(\d+)'
WEB_BUILDER = (
r'dart2js-(ie9|ie10|ie11|ff|safari|chrome|chromeOnAndroid|safarimobilesim|opera|drt)-(win7|win8|mac10\.7|mac10\.8|mac10\.9|linux)(-(all|html))?(-(csp))?(-(\d+)-(\d+))?')
IE_VERSIONS = ['ie10', 'ie11']
DART2JS_FULL_CONFIGURATIONS = {
'linux' : [
{'runtime' : 'ff'},
{'runtime' : 'chrome'},
],
'mac' : [ ],
'windows-ie10' : [
{'runtime' : 'ie10'},
{'runtime' : 'chrome'},
],
'windows-ie11' : [
{'runtime' : 'ie11'},
{'runtime' : 'ff'},
],
}
def GetBuildInfo(builder_name, is_buildbot):
"""Returns a BuildInfo object for the current buildbot based on the
name of the builder.
"""
compiler = None
runtime = None
mode = None
system = None
checked = False
host_checked = False
minified = False
shard_index = None
total_shards = None
test_set = None
csp = None
arch = None
dart2js_full = False
batch = True
builder_tag = None
cps_ir = None
dart2js_pattern = re.match(DART2JS_BUILDER, builder_name)
dart2js_full_pattern = re.match(DART2JS_FULL_BUILDER, builder_name)
web_pattern = re.match(WEB_BUILDER, builder_name)
dartium_pattern = re.match(DARTIUM_BUILDER, builder_name)
if web_pattern:
compiler = 'dart2js'
runtime = web_pattern.group(1)
system = web_pattern.group(2)
mode = 'release'
test_set = web_pattern.group(4)
if web_pattern.group(6) == 'csp':
csp = True
# Always run csp mode minified
minified = True
shard_index = web_pattern.group(8)
total_shards = web_pattern.group(9)
elif dart2js_full_pattern:
mode = 'release'
compiler = 'dart2js'
dart2js_full = True
system = dart2js_full_pattern.group(1)
# windows-ie10 or windows-ie11 means a windows machine with that respective
# version of ie installed. There is no difference in how we handle testing.
# We use the builder tag to pass along this information.
if system.startswith('win'):
ie = dart2js_full_pattern.group(3)
assert ie in IE_VERSIONS
builder_tag = 'windows-%s' % ie
system = 'windows'
if dart2js_full_pattern.group(4):
checked = True
if dart2js_full_pattern.group(5):
minified = True
shard_index = dart2js_full_pattern.group(6)
total_shards = dart2js_full_pattern.group(7)
elif dart2js_pattern:
compiler = 'dart2js'
system = dart2js_pattern.group(1)
runtime = 'd8'
arch = 'ia32'
if dart2js_pattern.group(3) == 'jsshell':
runtime = 'jsshell'
mode = dart2js_pattern.group(4)
# The valid naming parts for checked and host-checked are:
# Empty: checked=False, host_checked=False
# -checked: checked=True, host_checked=False
# -host-checked: checked=False, host_checked=True
# -checked-host-checked: checked=True, host_checked=True
if dart2js_pattern.group(6) == 'checked':
checked = True
if dart2js_pattern.group(6) == 'host-checked':
host_checked = True
if dart2js_pattern.group(8) == 'host-checked':
host_checked = True
if dart2js_pattern.group(10) == 'minified':
minified = True
if dart2js_pattern.group(12) == 'x64':
arch = 'x64'
if dart2js_pattern.group(14) == 'cps':
cps_ir = True
shard_index = dart2js_pattern.group(15)
total_shards = dart2js_pattern.group(16)
elif dartium_pattern:
compiler = 'none'
runtime = 'dartium'
mode = 'release'
system = dartium_pattern.group(1)
else :
return None
# We have both win7 and win8 bots, functionality is the same.
if system.startswith('win'):
system = 'windows'
# We have both 10.8 and 10.9 bots, functionality is the same.
if system == 'mac10.8' or system == 'mac10.9':
system = 'mac'
if (system == 'windows' and platform.system() != 'Windows') or (
system == 'mac' and platform.system() != 'Darwin') or (
system == 'linux' and platform.system() != 'Linux'):
print ('Error: You cannot emulate a buildbot with a platform different '
'from your own.')
return None
return bot.BuildInfo(compiler, runtime, mode, system, checked, host_checked,
minified, shard_index, total_shards, is_buildbot,
test_set, csp, arch, dart2js_full, batch=batch,
builder_tag=builder_tag, cps_ir=cps_ir)
def NeedsXterm(compiler, runtime):
return runtime in ['ie9', 'ie10', 'ie11', 'chrome', 'safari', 'opera',
'ff', 'drt', 'dartium']
def TestStepName(name, runtime, flags):
# Filter out flags with '=' as this breaks the /stats feature of the
# build bot.
flags = [x for x in flags if not '=' in x]
step_name = '%s-%s tests %s' % (name, runtime, ' '.join(flags))
return step_name.strip()
IsFirstTestStepCall = True
def TestStep(name, mode, system, compiler, runtime, targets, flags, arch):
step_name = TestStepName(name, runtime, flags)
with bot.BuildStep(step_name, swallow_error=True):
sys.stdout.flush()
if NeedsXterm(compiler, runtime) and system == 'linux':
cmd = ['xvfb-run', '-a', '--server-args=-screen 0 1024x768x24']
else:
cmd = []
user_test = os.environ.get('USER_TEST', 'no')
cmd.extend([sys.executable,
os.path.join(os.curdir, 'tools', 'test.py'),
'--step_name=' + step_name,
'--mode=' + mode,
'--compiler=' + compiler,
'--runtime=' + runtime,
'--arch=' + arch,
'--time',
'--use-sdk',
'--report',
'--write-debug-log',
'--write-test-outcome-log'])
if user_test == 'yes':
cmd.append('--progress=color')
else:
cmd.extend(['--progress=buildbot', '-v'])
cmd.append('--reset-browser-configuration')
global IsFirstTestStepCall
if IsFirstTestStepCall:
IsFirstTestStepCall = False
else:
cmd.append('--append_logs')
if flags:
cmd.extend(flags)
cmd.extend(targets)
print 'Running: %s' % (' '.join(map(lambda arg: '"%s"' % arg, cmd)))
sys.stdout.flush()
bot.RunProcess(cmd)
def TestCompiler(runtime, mode, system, flags, is_buildbot, arch,
compiler=None, dart2js_full=False):
""" test the compiler.
Args:
- runtime: either 'd8', 'jsshell', or one of the browsers, see GetBuildInfo
- mode: either 'debug' or 'release'
- system: either 'linux', 'mac', 'windows'
- flags: extra flags to pass to test.dart
- is_buildbot: true if we are running on a real buildbot instead of
emulating one.
- arch: The architecture to run on.
- compiler: The compiler to use for test.py (default is 'dart2js').
"""
if not compiler:
compiler = 'dart2js'
def GetPath(runtime):
""" Helper to get the path to the Chrome or Firefox executable for a
particular platform on the buildbot. Throws a KeyError if runtime is not
either 'chrome' or 'ff'."""
if system == 'mac':
partDict = {'chrome': 'Google\\ Chrome', 'ff': 'Firefox'}
mac_path = '/Applications/%s.app/Contents/MacOS/%s'
path_dict = {'chrome': mac_path % (partDict[runtime], partDict[runtime]),
'ff': mac_path % (partDict[runtime], partDict[runtime].lower())}
elif system == 'linux':
path_dict = {'ff': 'firefox', 'chrome': 'google-chrome'}
else:
# Windows.
path_dict = {'ff': os.path.join('C:/', 'Program Files (x86)',
'Mozilla Firefox', 'firefox.exe'),
'chrome': os.path.join('C:/', 'Users', 'chrome-bot', 'AppData',
'Local', 'Google', 'Chrome', 'Application', 'chrome.exe')}
return path_dict[runtime]
if (compiler == 'dart2js' and (runtime == 'ff' or runtime == 'chrome')
and is_buildbot):
# Print out browser version numbers if we're running on the buildbot (where
# we know the paths to these browser installations).
version_query_string = '"%s" --version' % GetPath(runtime)
if runtime == 'ff' and system == 'windows':
version_query_string += '| more'
elif runtime == 'chrome' and system == 'windows':
version_query_string = ('''reg query "HKCU\\Software\\Microsoft\\''' +
'''Windows\\CurrentVersion\\Uninstall\\Google Chrome" /v Version''')
p = subprocess.Popen(version_query_string,
stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
output, stderr = p.communicate()
output = output.split()
try:
print 'Version of %s: %s' % (runtime, output[-1])
except IndexError:
# Failed to obtain version information. Continue running tests.
pass
unit_test_flags = [flag for flag in flags if flag.startswith('--shard')]
# Run the unit tests in checked mode (the VM's checked mode).
unit_test_flags.append('--checked')
if runtime == 'd8':
# The dart2js compiler isn't self-hosted (yet) so we run its
# unit tests on the VM. We avoid doing this on the builders
# that run the browser tests to cut down on the cycle time.
TestStep("dart2js_unit", mode, system, 'none', 'vm', ['dart2js', 'try'],
unit_test_flags, arch)
if compiler == 'dart2js' and runtime == 'drt':
# Ensure that we run the "try" tests on Content Shell.
TestStep("incremental_compilation", mode, system, 'none', runtime,
['try'], unit_test_flags, arch)
if compiler == 'dart2js' and runtime in ['ie10', 'ie11']:
TestStep(compiler, mode, system, compiler, runtime,
['html', 'pkg', 'samples', 'co19'], flags, arch)
else:
# Run the default set of test suites.
TestStep(compiler, mode, system, compiler,
runtime, [], flags, arch)
if compiler == 'dart2js':
# TODO(kasperl): Consider running peg and css tests too.
extras = ['dart2js_extra', 'dart2js_native']
extras_flags = flags
if (system == 'linux'
and runtime == 'd8'
and not '--host-checked' in extras_flags):
# Run the extra tests in checked mode, but only on linux/d8.
# Other systems have less resources and tend to time out.
extras_flags = extras_flags + ['--host-checked']
TestStep('dart2js_extra', mode, system, 'dart2js', runtime, extras,
extras_flags, arch)
if mode == 'release':
TestStep('try_dart', mode, system, 'dart2js', runtime, ['try'],
extras_flags, arch)
def GetHasHardCodedCheckedMode(build_info):
# TODO(ricow): We currently run checked mode tests on chrome on linux and
# on the slow (all) IE windows bots. This is a hack and we should use the
# normal sharding and checked splitting functionality when we get more
# vms for testing this.
if (build_info.system == 'linux' and build_info.runtime == 'drt'):
return True
if build_info.runtime.startswith('ie') and build_info.test_set == 'all':
return True
return False
def GetLocalIPAddress():
hostname = socket.gethostname()
# '$ host chromeperf02' results for example in
# 'chromeperf02.perf.chromium.org has address 172.22.28.55'
output = subprocess.check_output(["host", hostname])
match = re.match(r'.*\s+([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\s+.*', output)
if not match:
raise Exception("Could not determine local ip address "
"(hostname: '%s', host command output: '%s')."
% (hostname, output))
return match.group(1)
def AddAndroidToolsToPath():
par_dir = os.path.pardir
join = os.path.join
dart_dir = join(os.path.dirname(__file__), par_dir, par_dir)
android_sdk = join(dart_dir, 'third_party', 'android_tools', 'sdk')
tools_dir = os.path.abspath(join(android_sdk, 'tools'))
platform_tools_dir = os.path.abspath(join(android_sdk, 'platform-tools'))
os.environ['PATH'] = os.pathsep.join(
[os.environ['PATH'], tools_dir, platform_tools_dir])
def RunCompilerTests(build_info):
test_flags = []
if build_info.shard_index:
test_flags = ['--shards=%s' % build_info.total_shards,
'--shard=%s' % build_info.shard_index]
if build_info.checked: test_flags += ['--checked']
if build_info.minified: test_flags += ['--minified']
if build_info.host_checked: test_flags += ['--host-checked']
if build_info.batch: test_flags += ['--dart2js-batch']
if build_info.builder_tag: test_flags += ['--builder-tag=' +
build_info.builder_tag]
if build_info.cps_ir: test_flags += ['--cps-ir']
if build_info.dart2js_full:
compiler = build_info.compiler
assert compiler == 'dart2js'
system = build_info.system
arch = build_info.arch
mode = build_info.mode
is_buildbot = build_info.is_buildbot
config = build_info.builder_tag if system == 'windows' else system
for configuration in DART2JS_FULL_CONFIGURATIONS[config]:
additional_flags = configuration.get('additional_flags', [])
TestCompiler(configuration['runtime'], mode, system,
test_flags + additional_flags, is_buildbot, arch,
compiler=compiler, dart2js_full=True)
else:
if build_info.csp: test_flags += ['--csp']
if build_info.runtime == 'chromeOnAndroid':
test_flags.append('--local_ip=%s' % GetLocalIPAddress())
# test.py expects the android tools directories to be in PATH
# (they contain for example 'adb')
AddAndroidToolsToPath()
TestCompiler(build_info.runtime, build_info.mode, build_info.system,
list(test_flags), build_info.is_buildbot,
build_info.arch, compiler=build_info.compiler)
# See comment in GetHasHardCodedCheckedMode, this is a hack.
if (GetHasHardCodedCheckedMode(build_info)):
TestCompiler(build_info.runtime, build_info.mode, build_info.system,
test_flags + ['--checked'], build_info.is_buildbot,
build_info.arch, compiler=build_info.compiler)
def BuildCompiler(build_info):
"""
Builds the SDK.
- build_info: the buildInfo object, containing information about what sort of
build and test to be run.
"""
with bot.BuildStep('Build SDK'):
target = 'dart2js_bot'
# Try-dart takes more than 20 min in debug mode and makes the bot time out.
# We use the debug target which does not include try
if build_info.mode == 'debug':
target = 'dart2js_bot_debug'
args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode,
'--arch=' + build_info.arch, target]
print 'Build SDK and d8: %s' % (' '.join(args))
bot.RunProcess(args)
if __name__ == '__main__':
bot.RunBot(GetBuildInfo, RunCompilerTests, build_step=BuildCompiler)

View file

@ -1,121 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import os
import re
import sys
import bot
GCS_BUCKET = 'gs://dart-cross-compiled-binaries'
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(SCRIPT_DIR, '..'))
import utils
CROSS_VM = r'cross-(arm)-vm-linux-(release)'
TARGET_VM = r'target-(arm)-vm-linux-(([0-9]+)-([0-9]+))?(release)'
GSUTIL = utils.GetBuildbotGSUtilPath()
def run(args):
print 'Running: %s' % (' '.join(args))
sys.stdout.flush()
bot.RunProcess(args)
def tarball_name(arch, mode, revision):
return 'cross_build_%s_%s_%s.tar.bz2' % (arch, mode, revision)
def record_names(name, arch, mode):
return ('record_%s_%s_%s.json' % (name, arch, mode),
'record_output_%s_%s_%s.json' % (name, arch, mode))
def cross_compiling_builder(arch, mode):
build_py = os.path.join('tools', 'build.py')
revision = os.environ['BUILDBOT_GOT_REVISION']
tarball = tarball_name(arch, mode, revision)
temporary_files = [tarball]
bot.Clobber()
try:
num_run = int(os.environ['BUILDBOT_ANNOTATED_STEPS_RUN'])
if num_run == 1:
with bot.BuildStep('Build %s %s' % (arch, mode)):
run([sys.executable, build_py,
'-m%s' % mode, '--arch=%s' % arch, 'runtime'])
with bot.BuildStep('Create build tarball'):
run(['tar', '-cjf', tarball, '--exclude=**/obj',
'--exclude=**/obj.host', '--exclude=**/obj.target',
'--exclude=**/*analyzer*', 'out/'])
with bot.BuildStep('Upload build tarball'):
uri = "%s/%s" % (GCS_BUCKET, tarball)
run([GSUTIL, 'cp', '-a', 'public-read', tarball, uri])
elif num_run == 2:
with bot.BuildStep('tests'):
print "Please see the target device for results."
print "We no longer record/replay tests."
else:
raise Exception("Invalid annotated steps run")
finally:
for path in temporary_files:
if os.path.exists(path):
os.remove(path)
def target_builder(arch, mode, total_shards, shard_index):
test_py = os.path.join('tools', 'test.py')
test_args = [sys.executable, test_py, '--progress=line', '--report',
'--time', '--compiler=none', '--runtime=vm', '--write-debug-log',
'--write-test-outcome-log', '--mode=' + mode, '--arch=' + arch]
if total_shards and shard_index:
test_args.append('--shards=%s' % total_shards)
test_args.append('--shard=%s' % shard_index)
revision = os.environ['BUILDBOT_GOT_REVISION']
tarball = tarball_name(arch, mode, revision)
temporary_files = [tarball]
bot.Clobber()
try:
with bot.BuildStep('Fetch build tarball'):
run([GSUTIL, 'cp', "%s/%s" % (GCS_BUCKET, tarball), tarball])
with bot.BuildStep('Unpack build tarball'):
run(['tar', '-xjf', tarball])
with bot.BuildStep('execute tests'):
run(test_args)
with bot.BuildStep('execute checked_tests'):
run(test_args + ['--checked', '--append_logs'])
finally:
for path in temporary_files:
if os.path.exists(path):
os.remove(path)
# We always clobber this to save disk on the arm board.
bot.Clobber(force=True)
def main():
name, is_buildbot = bot.GetBotName()
cross_vm_pattern_match = re.match(CROSS_VM, name)
target_vm_pattern_match = re.match(TARGET_VM, name)
if cross_vm_pattern_match:
arch = cross_vm_pattern_match.group(1)
mode = cross_vm_pattern_match.group(2)
cross_compiling_builder(arch, mode)
elif target_vm_pattern_match:
arch = target_vm_pattern_match.group(1)
mode = target_vm_pattern_match.group(5)
shard_index = target_vm_pattern_match.group(3)
total_shards = target_vm_pattern_match.group(4)
target_builder(arch, mode, total_shards, shard_index)
else:
raise Exception("Unknown builder name %s" % name)
if __name__ == '__main__':
try:
sys.exit(main())
except OSError as e:
sys.exit(e.errno)

View file

@ -1,89 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""
Dartium on Android buildbot steps.
Runs steps after the buildbot builds Dartium on Android,
which should upload the APK to an attached device, and run
Dart and chromium tests on it.
"""
import optparse
import os
import string
import subprocess
import sys
import bot
import bot_utils
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(os.path.join(SCRIPT_DIR, '..'))
import utils
CS_LOCATION = 'apks/ContentShell.apk'
def GetOptionsParser():
parser = optparse.OptionParser("usage: %prog [options]")
parser.add_option("--build-products-dir",
help="The directory containing the products of the build.")
return parser
def UploadSetACL(gsutil, local, remote):
gsutil.upload(local, remote, public=True)
def UploadAPKs(options):
with bot.BuildStep('Upload apk'):
bot_name = os.environ.get('BUILDBOT_BUILDERNAME')
on_fyi = 'fyi-' in os.environ.get('BUILDBOT_SCHEDULER')
if '-integration' in bot_name or on_fyi:
return
channel = bot_utils.GetChannelFromName(bot_name)
namer = bot_utils.GCSNamer(channel=channel)
gsutil = bot_utils.GSUtil()
web_link_prefix = 'https://storage.cloud.google.com/'
# Archive content shell
local = os.path.join(options.build_products_dir, CS_LOCATION)
for revision in [utils.GetArchiveVersion(), 'latest']:
# TODO(whesse): pass in arch and mode from reciepe
remote = namer.dartium_android_apk_filepath(revision,
'content_shell-android',
'arm',
'release')
content_shell_link = string.replace(remote, 'gs://', web_link_prefix)
UploadSetACL(gsutil, local, remote)
print "Uploaded content shell, available from: %s" % content_shell_link
def RunContentShellTests(options):
with bot.BuildStep('ContentShell tests'):
subprocess.check_call([os.path.join(SCRIPT_DIR, 'run_android_tests.sh'),
os.path.join(options.build_products_dir, CS_LOCATION)])
def main():
if sys.platform != 'linux2':
print "This script was only tested on linux. Please run it on linux!"
sys.exit(1)
parser = GetOptionsParser()
(options, args) = parser.parse_args()
if not options.build_products_dir:
print "No build products directory given."
sys.exit(1)
UploadAPKs(options)
RunContentShellTests(options)
sys.exit(0)
if __name__ == '__main__':
main()

View file

@ -1,49 +0,0 @@
#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Fetches an archived chromium build into
src/chrome/tools/test/reference_build unless
src/chrome/tools/test/reference_build/REQUESTED_REVISION is the same as
src/chrome/tools/test/reference_build/CURRENT_REVISION.
Must be run from the root of a Dartium or multivm checkout.
Usage:
$ ./src/dart/tools/bots/fetch_reference_build_revision.py
"""
import os
import subprocess
import sys
def main(argv):
dirname = os.path.join('src', 'chrome', 'tools',
'test', 'reference_build')
request = os.path.join(dirname, 'REQUESTED_REVISION')
found = os.path.join(dirname, 'CURRENT_REVISION')
if not os.path.exists(request):
return
with file(request, 'r') as f:
request_revision = f.read()
if os.path.exists(found):
with file(found, 'r') as f:
found_revision = f.read()
if found_revision == request_revision:
return
get_script = os.path.join('src', 'dart', 'tools',
'bots', 'get_chromium_build.py')
get_script = os.path.abspath(get_script)
exit_code = subprocess.call(['python', get_script,
'-r', request_revision,
'-t', dirname])
if exit_code == 0:
with file(found, 'w') as f:
f.write(request_revision)
return exit_code
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,171 +0,0 @@
#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Gets a Chromium archived build, and unpacks it
into a target directory.
Use -r option to specify the revison number
Use -t option to specify the directory to unzip the build into.
Usage:
$ get_chromium_build.py -r <revision> -t <target>
"""
import logging
import optparse
import os
import platform
import shutil
import subprocess
import sys
import time
import urllib
import urllib2
import zipfile
# Example chromium build location:
# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
'chromium-browser-snapshots/%s/%s/%s')
class BuildUpdater(object):
_PLATFORM_PATHS_MAP = {
'Linux': { 'zipfiles': ['chrome-linux.zip'],
'folder': 'chrome_linux',
'archive_path': 'Linux_x64'},
'Darwin': {'zipfiles': ['chrome-mac.zip'],
'folder': 'chrome_mac',
'archive_path': 'Mac'},
'Windows': {'zipfiles': ['chrome-win32.zip',
'chrome-win32-syms.zip'],
'folder': 'chrome_win',
'archive_path': 'Win'}}
def __init__(self, options):
platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
self._zipfiles = platform_data['zipfiles']
self._folder = platform_data['folder']
self._archive_path = platform_data['archive_path']
self._revision = int(options.revision)
self._target_dir = options.target_dir
self._download_dir = os.path.join(self._target_dir, 'downloads')
def _GetBuildUrl(self, revision, filename):
return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)
def _FindBuildRevision(self, revision, filename):
MAX_REVISIONS_PER_BUILD = 100
for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
if self._DoesBuildExist(revision_guess, filename):
return revision_guess
else:
time.sleep(.1)
return None
def _DoesBuildExist(self, revision_guess, filename):
url = self._GetBuildUrl(revision_guess, filename)
r = urllib2.Request(url)
r.get_method = lambda: 'HEAD'
try:
urllib2.urlopen(r)
return True
except urllib2.HTTPError, err:
if err.code == 404:
return False
def _DownloadBuild(self):
if not os.path.exists(self._download_dir):
os.makedirs(self._download_dir)
for zipfile in self._zipfiles:
build_revision = self._FindBuildRevision(self._revision, zipfile)
if not build_revision:
logging.critical('Failed to find %s build for r%s\n',
self._archive_path,
self._revision)
sys.exit(1)
url = self._GetBuildUrl(build_revision, zipfile)
logging.info('Downloading %s', url)
r = urllib2.urlopen(url)
with file(os.path.join(self._download_dir, zipfile), 'wb') as f:
f.write(r.read())
def _UnzipFile(self, dl_file, dest_dir):
if not zipfile.is_zipfile(dl_file):
return False
logging.info('Unzipping %s', dl_file)
with zipfile.ZipFile(dl_file, 'r') as z:
for content in z.namelist():
dest = os.path.join(dest_dir, content[content.find('/')+1:])
# Create dest parent dir if it does not exist.
if not os.path.isdir(os.path.dirname(dest)):
logging.info('Making %s', dest)
os.makedirs(os.path.dirname(dest))
# If dest is just a dir listing, do nothing.
if not os.path.basename(dest):
continue
with z.open(content) as unzipped_content:
logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
with file(dest, 'wb') as dest_file:
dest_file.write(unzipped_content.read())
permissions = z.getinfo(content).external_attr >> 16
if permissions:
os.chmod(dest, permissions)
return True
def _ClearDir(self, dir):
"""Clears all files in |dir| except for hidden files and folders."""
for root, dirs, files in os.walk(dir):
# Skip hidden files and folders (like .svn and .git).
files = [f for f in files if f[0] != '.']
dirs[:] = [d for d in dirs if d[0] != '.']
for f in files:
os.remove(os.path.join(root, f))
def _ExtractBuild(self):
dest_dir = os.path.join(self._target_dir, self._folder)
self._ClearDir(dest_dir)
for root, _, dl_files in os.walk(os.path.join(self._download_dir)):
for dl_file in dl_files:
dl_file = os.path.join(root, dl_file)
if not self._UnzipFile(dl_file, dest_dir):
logging.info('Copying %s to %s', dl_file, dest_dir)
shutil.copy(dl_file, dest_dir)
shutil.rmtree(self._download_dir)
def DownloadAndUpdateBuild(self):
self._DownloadBuild()
self._ExtractBuild()
def ParseOptions(argv):
parser = optparse.OptionParser()
usage = 'usage: %prog <options>'
parser.set_usage(usage)
parser.add_option('-r', dest='revision',
help='Revision to download.')
parser.add_option('-t', dest='target_dir',
help='Target directory for unzipped Chromium.')
(options, _) = parser.parse_args(argv)
if not options.revision:
logging.critical('Must specify -r.\n')
sys.exit(1)
if not options.target_dir:
logging.critical('Must specify -t.\n')
sys.exit(1)
return options
def main(argv):
logging.getLogger().setLevel(logging.DEBUG)
options = ParseOptions(argv)
b = BuildUpdater(options)
b.DownloadAndUpdateBuild()
logging.info('Successfully got archived Chromium build.')
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,31 +0,0 @@
#!/usr/bin/env python
#
# Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import os
import os.path
import shutil
import sys
import subprocess
SCRIPT_DIR = os.path.dirname(sys.argv[0])
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..', '..'))
def main(argv):
os.environ["DART_USE_GYP"] = "1"
generate_buildfiles = os.path.join(
DART_ROOT, 'tools', 'generate_buildfiles.py')
gclient_result = subprocess.call(['python', generate_buildfiles])
if gclient_result != 0:
return gclient_result
build_py = os.path.join(DART_ROOT, 'tools', 'build.py')
build_result = subprocess.call(['python', build_py] + argv[1:])
if build_result != 0:
return build_result
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,38 +0,0 @@
#!/usr/bin/env python
#
# Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import os
import os.path
import shutil
import sys
import subprocess
SCRIPT_DIR = os.path.dirname(sys.argv[0])
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..', '..'))
def main(argv):
test_py = os.path.join(DART_ROOT, 'tools', 'test.py')
build_result = subprocess.call(
['python', test_py, '--builder-tag=no_ipv6']
+ argv[1:])
if build_result != 0:
return build_result
build_result = subprocess.call(
['python', test_py, '--builder-tag=no_ipv6',
'-cdart2js', '-rd8', '--use-sdk', 'language/first']
+ argv[1:])
if build_result != 0:
return build_result
build_result = subprocess.call(
['python', test_py, '--builder-tag=no_ipv6',
'-cdart2analyzer', '-rnone', '--use-sdk', 'language/first']
+ argv[1:])
if build_result != 0:
return build_result
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,31 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import re
import sys
import bot
import bot_utils
utils = bot_utils.GetUtils()
PACKAGES_BUILDER = r'packages-(windows|linux|mac)-(core-elements|polymer)'
def PackagesConfig(name, is_buildbot):
packages_pattern = re.match(PACKAGES_BUILDER, name)
if not packages_pattern:
return None
system = packages_pattern.group(1)
return bot.BuildInfo('none', 'vm', 'release', system, checked=True)
def PackagesSteps(build_info):
with bot.BuildStep('Testing packages'):
bot_name, _ = bot.GetBotName()
print bot_name
if __name__ == '__main__':
bot.RunBot(PackagesConfig, PackagesSteps)

View file

@ -1,62 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
"""
Pkg buildbot steps.
Runs tests for packages that are hosted in the main Dart repo and in
third_party/pkg_tested.
"""
import os
import re
import sys
import bot
PKG_BUILDER = r'pkg-(linux|mac|win)(-(russian))?'
def PkgConfig(name, is_buildbot):
"""Returns info for the current buildbot based on the name of the builder.
Currently, this is just:
- system: "linux", "mac", or "win"
"""
pkg_pattern = re.match(PKG_BUILDER, name)
if not pkg_pattern:
return None
system = pkg_pattern.group(1)
locale = pkg_pattern.group(3)
if system == 'win': system = 'windows'
return bot.BuildInfo('none', 'vm', 'release', system, checked=True,
builder_tag=locale)
def PkgSteps(build_info):
common_args = ['--write-test-outcome-log']
if build_info.builder_tag:
common_args.append('--builder-tag=%s' % build_info.builder_tag)
# There are a number of big/integration tests in pkg, run with bigger timeout
common_args.append('--timeout=120')
# We have some unreproducible vm crashes on these bots
common_args.append('--copy-coredumps')
bot.RunTest('pkg', build_info,
common_args + ['pkg', 'docs'],
swallow_error=True)
with bot.BuildStep('third_party pkg tests', swallow_error=True):
pkg_tested = os.path.join('third_party', 'pkg_tested')
for entry in os.listdir(pkg_tested):
path = os.path.join(pkg_tested, entry)
if os.path.isdir(path):
bot.RunTestRunner(build_info, path)
if __name__ == '__main__':
bot.RunBot(PkgConfig, PkgSteps)

View file

@ -1,29 +0,0 @@
#!/usr/bin/env python
#
# Copyright (c) 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Writes a revision number into src/chrome/tools/test/reference_build/REVISON
Must be run from the root of a Dartium or multivm checkout.
Usage:
$ ./src/dart/tools/bots/set_reference_build_revision.py <revision>
"""
import os
import sys
def main(argv):
revision = argv[1]
output = os.path.join('src', 'chrome', 'tools',
'test', 'reference_build',
'REQUESTED_REVISION')
dirname = os.path.dirname(output)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
with file(output, 'w') as f:
f.write(revision)
if __name__ == '__main__':
sys.exit(main(sys.argv))

View file

@ -1,12 +0,0 @@
#!/usr/bin/python
# Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import sys
def main():
sys.exit(0)
if __name__ == '__main__':
main()