diff --git a/DEPS b/DEPS index 72dacdafda1..43e3f659b9d 100644 --- a/DEPS +++ b/DEPS @@ -82,7 +82,7 @@ vars = { "ply_rev": "@604b32590ffad5cbb82e4afef1d305512d06ae93", "plugin_tag": "@0.1.0", "pool_rev": "@22e12aeb16ad0b626900dbe79e4a25391ddfb28c", - "pub_rev": "@68308887177ef3b78fdd52f068e4eaefe1b17e07", + "pub_rev": "@9d707158fedc86fc2b02f62cdfe804902b098d9d", "pub_semver_tag": "@1.2.1", "scheduled_test_tag": "@0.11.8+1", "shelf_rev": "@1e87b79b21ac5e6fa2f93576d6c06eaa65285ef4", @@ -240,7 +240,7 @@ deps = { (Var("github_mirror") % "pool") + Var("pool_rev"), Var("dart_root") + "/third_party/pkg/pub_semver": (Var("github_mirror") % "pub_semver") + Var("pub_semver_tag"), - Var("dart_root") + "/third_party/pkg_tested/pub": + Var("dart_root") + "/third_party/pkg/pub": ("https://github.com/dart-lang/pub.git") + Var("pub_rev"), Var("dart_root") + "/third_party/pkg/scheduled_test": (Var("github_mirror") % "scheduled_test") + diff --git a/client/tools/buildbot_annotated_steps.py b/client/tools/buildbot_annotated_steps.py index 1ca39335a4f..bde2a63750a 100755 --- a/client/tools/buildbot_annotated_steps.py +++ b/client/tools/buildbot_annotated_steps.py @@ -79,7 +79,9 @@ def main(): return status name = GetName() - if name.startswith('pub-'): + if name.startswith('pkg-'): + status = ProcessBot(name, 'pkg') + elif name.startswith('pub-'): status = ProcessBot(name, 'pub') elif name.startswith('vm-android'): status = ProcessBot(name, 'android') diff --git a/sdk/bin/pub b/sdk/bin/pub index 96cc0dfae5b..e92773182e6 100755 --- a/sdk/bin/pub +++ b/sdk/bin/pub @@ -53,5 +53,5 @@ DART="$BUILD_DIR/dart-sdk/bin/dart" PACKAGES_DIR="$BUILD_DIR/packages/" # Run pub. -PUB="$SDK_DIR/../third_party/pkg_tested/pub/bin/pub.dart" +PUB="$SDK_DIR/../third_party/pkg/pub/bin/pub.dart" exec "$DART" "${VM_OPTIONS[@]}" "--package-root=$PACKAGES_DIR" "$PUB" "$@" diff --git a/sdk/bin/pub.bat b/sdk/bin/pub.bat index d64b492a985..1326b57de03 100644 --- a/sdk/bin/pub.bat +++ b/sdk/bin/pub.bat @@ -35,7 +35,7 @@ set PACKAGES_DIR=%BUILD_DIR%\packages set DART=%BUILD_DIR%\dart-sdk\bin\dart rem Run pub. -set PUB="%SDK_DIR%\..\third_party\pkg_tested\pub\bin\pub.dart" +set PUB="%SDK_DIR%\..\third_party\pkg\pub\bin\pub.dart" "%DART%" %VM_OPTIONS% --package-root="%PACKAGES_DIR%" "%PUB%" %* endlocal diff --git a/tools/bots/pkg.py b/tools/bots/pkg.py new file mode 100644 index 00000000000..1ba85bb78c5 --- /dev/null +++ b/tools/bots/pkg.py @@ -0,0 +1,83 @@ +#!/usr/bin/python + +# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +""" +Pkg buildbot steps. + +Runs tests for packages that are hosted in the main Dart repo and in +third_party/pkg_tested. +""" + +import re +import sys + +import bot + +PKG_BUILDER = r'pkg-(linux|mac|win)(-(russian))?(-(debug))?' + +def PkgConfig(name, is_buildbot): + """Returns info for the current buildbot based on the name of the builder. + + Currently, this is just: + - mode: "debug", "release" + - system: "linux", "mac", or "win" + """ + pkg_pattern = re.match(PKG_BUILDER, name) + if not pkg_pattern: + return None + + system = pkg_pattern.group(1) + locale = pkg_pattern.group(3) + mode = pkg_pattern.group(5) or 'release' + if system == 'win': system = 'windows' + + return bot.BuildInfo('none', 'vm', mode, system, checked=True, + builder_tag=locale) + +def PkgSteps(build_info): + with bot.BuildStep('Build package-root'): + args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode, + 'packages'] + print 'Building package-root: %s' % (' '.join(args)) + bot.RunProcess(args) + + common_args = ['--write-test-outcome-log'] + if build_info.builder_tag: + common_args.append('--builder-tag=%s' % build_info.builder_tag) + + # There are a number of big/integration tests in pkg, run with bigger timeout + common_args.append('--timeout=120') + # We have some unreproducible vm crashes on these bots + common_args.append('--copy-coredumps') + + # We are seeing issues with pub get calls on the windows bots. + # Experiment with not running concurrent calls. + if build_info.system == 'windows': + common_args.append('-j1') + if build_info.mode == 'release': + bot.RunTest('pkg ', build_info, + common_args + ['pkg', 'docs', 'pkg_tested'], + swallow_error=True) + else: + # Pkg tests currently have a lot of timeouts when run in debug mode. + # See issue 18479 + bot.RunTest('pkg', build_info, common_args + ['pkg', 'docs'], + swallow_error=True) + + if build_info.mode == 'release': + pkgbuild_build_info = bot.BuildInfo('none', 'vm', build_info.mode, + build_info.system, checked=False) + bot.RunTest('pkgbuild_repo_pkgs', pkgbuild_build_info, + common_args + ['--append_logs', '--use-repository-packages', + 'pkgbuild'], + swallow_error=True) + + public_args = (common_args + + ['--append_logs', '--use-public-packages', 'pkgbuild']) + bot.RunTest('pkgbuild_public_pkgs', pkgbuild_build_info, public_args) + +if __name__ == '__main__': + bot.RunBot(PkgConfig, PkgSteps) diff --git a/tools/bots/pub.py b/tools/bots/pub.py index 98ae6720c63..9d144568817 100755 --- a/tools/bots/pub.py +++ b/tools/bots/pub.py @@ -10,73 +10,66 @@ Pub buildbot steps. Runs tests for pub and the pub packages that are hosted in the main Dart repo. """ +import os import re +import shutil import sys import bot +import bot_utils -PUB_BUILDER = r'pub-(linux|mac|win)(-(russian))?(-(debug))?' +utils = bot_utils.GetUtils() + +BUILD_OS = utils.GuessOS() + +PUB_BUILDER = r'pub-(linux|mac|win)' def PubConfig(name, is_buildbot): """Returns info for the current buildbot based on the name of the builder. Currently, this is just: - - mode: "debug", "release" + - mode: always release, we don't run pub in debug mode - system: "linux", "mac", or "win" + - checked: always true """ pub_pattern = re.match(PUB_BUILDER, name) if not pub_pattern: return None system = pub_pattern.group(1) - locale = pub_pattern.group(3) - mode = pub_pattern.group(5) or 'release' + mode = 'release' if system == 'win': system = 'windows' - return bot.BuildInfo('none', 'vm', mode, system, checked=True, - builder_tag=locale) + return bot.BuildInfo('none', 'vm', mode, system, checked=True) + +def Run(command): + print "Running %s" % ' '.join(command) + return bot.RunProcess(command) def PubSteps(build_info): - with bot.BuildStep('Build package-root'): - args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode, - 'packages'] - print 'Building package-root: %s' % (' '.join(args)) - bot.RunProcess(args) + sdk_bin = os.path.join( + bot_utils.DART_DIR, + utils.GetBuildSdkBin(BUILD_OS, build_info.mode, build_info.arch)) + pub_script_name = 'pub.bat' if build_info.system == 'windows' else 'pub' + pub_bin = os.path.join(sdk_bin, pub_script_name) - common_args = ['--write-test-outcome-log'] - if build_info.builder_tag: - common_args.append('--builder-tag=%s' % build_info.builder_tag) + pub_copy = os.path.join(utils.GetBuildRoot('linux'), 'pub_copy') + pub_location = os.path.join('third_party', 'pkg', 'pub') - # There are a number of big/integration tests in pkg, run with bigger timeout - common_args.append('--timeout=120') - # We have some unreproducible vm crashes on these bots - common_args.append('--copy-coredumps') + with bot.BuildStep('Make copy of pub for testing'): + print 'Removing old copy %s' % pub_copy + shutil.rmtree(pub_copy, ignore_errors=True) + print 'Copying %s to %s' % (pub_location, pub_copy) + shutil.copytree(pub_location, pub_copy) - if build_info.system == 'windows': - common_args.append('-j1') - if build_info.mode == 'release': - bot.RunTest('pub and pkg ', build_info, - common_args + ['pub', 'pkg', 'docs', 'pkg_tested'], - swallow_error=True) - else: - # Pub tests currently have a lot of timeouts when run in debug mode. - # See issue 18479 - bot.RunTest('pub and pkg', build_info, common_args + ['pkg', 'docs'], - swallow_error=True) + # TODO(nweiz): add logic for testing pub. + with bot.BuildStep('Doing the magic ls'): + with utils.ChangedWorkingDirectory(pub_copy): + Run(['ls', '-l']) - if build_info.mode == 'release': - pkgbuild_build_info = bot.BuildInfo('none', 'vm', build_info.mode, - build_info.system, checked=False) - bot.RunTest('pkgbuild_repo_pkgs', pkgbuild_build_info, - common_args + ['--append_logs', '--use-repository-packages', - 'pkgbuild'], - swallow_error=True) + with bot.BuildStep('Running pub'): + Run([pub_bin, '--version']) - # We are seeing issues with pub get calls on the windows bots. - # Experiment with not running concurrent calls. - public_args = (common_args + - ['--append_logs', '--use-public-packages', 'pkgbuild']) - bot.RunTest('pkgbuild_public_pkgs', pkgbuild_build_info, public_args) if __name__ == '__main__': bot.RunBot(PubConfig, PubSteps) diff --git a/tools/create_sdk.py b/tools/create_sdk.py index 0372f1ef639..7ebe084f5cf 100755 --- a/tools/create_sdk.py +++ b/tools/create_sdk.py @@ -229,7 +229,7 @@ def Main(): RESOURCE = join(SDK_tmp, 'lib', '_internal', 'pub', 'asset') os.makedirs(os.path.dirname(RESOURCE)) - copytree(join(HOME, 'third_party', 'pkg_tested', 'pub', 'lib', 'src', + copytree(join(HOME, 'third_party', 'pkg', 'pub', 'lib', 'src', 'asset'), join(RESOURCE), ignore=ignore_patterns('.svn')) diff --git a/tools/deps/dartium.deps/DEPS b/tools/deps/dartium.deps/DEPS index b88d2f1acc5..65c121db4c1 100644 --- a/tools/deps/dartium.deps/DEPS +++ b/tools/deps/dartium.deps/DEPS @@ -34,7 +34,7 @@ vars.update({ "path_rev": "@b657c0854d1cf41c014986fa9d2321f1173df805", "plugin_tag": "@0.1.0", "pool_rev": "@22e12aeb16ad0b626900dbe79e4a25391ddfb28c", - "pub_rev": "@6f2a1b90b8210a85a38aab1af479c047681c29e6", + "pub_rev": "@9d707158fedc86fc2b02f62cdfe804902b098d9d", "pub_semver_tag": "@1.2.0", "shelf_rev": "@1e87b79b21ac5e6fa2f93576d6c06eaa65285ef4", "shelf_web_socket_rev": "@ff170cec2c0e4e5722cdf47c557be63b5035a602", @@ -109,7 +109,7 @@ deps.update({ "https://github.com/dart-lang/plugin.git" + Var("plugin_tag"), "src/dart/third_party/pkg/pool": "https://github.com/dart-lang/pool.git" + Var("pool_rev"), - "src/dart/third_party/pkg_tested/pub": + "src/dart/third_party/pkg/pub": "https://github.com/dart-lang/pub.git" + Var("pub_rev"), "src/dart/third_party/pkg/pub_semver": "https://github.com/dart-lang/pub_semver.git" + Var("pub_semver_tag"), diff --git a/tools/run_pub.py b/tools/run_pub.py index 37b6772ebf8..f54a4117016 100755 --- a/tools/run_pub.py +++ b/tools/run_pub.py @@ -12,7 +12,7 @@ import sys SCRIPT_DIR = os.path.dirname(sys.argv[0]) DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) -PUB_PATH = os.path.join(DART_ROOT, 'third_party/pkg_tested/pub/bin/pub.dart') +PUB_PATH = os.path.join(DART_ROOT, 'third_party/pkg/pub/bin/pub.dart') CANARY_PATH = os.path.join(DART_ROOT, 'tools', 'canary.dart') usage = """run_pub.py --package-root=""" diff --git a/utils/pub/pub.gyp b/utils/pub/pub.gyp index 252a255a32d..81dcec762fd 100644 --- a/utils/pub/pub.gyp +++ b/utils/pub/pub.gyp @@ -30,7 +30,7 @@ '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)', '--package-root=<(PRODUCT_DIR)/packages/', '--snapshot=<(SHARED_INTERMEDIATE_DIR)/pub.dart.snapshot', - '../../third_party/pkg_tested/pub/bin/pub.dart', + '../../third_party/pkg/pub/bin/pub.dart', ] }, ],