mirror of
https://github.com/dart-lang/sdk
synced 2024-10-06 16:31:07 +00:00
03730d9acc
Bug: https://github.com/dart-lang/sdk/issues/35360 Change-Id: Ief81cc626ab10095de000a97373357e447576618 Reviewed-on: https://dart-review.googlesource.com/c/86761 Reviewed-by: William Hesse <whesse@google.com>
360 lines
14 KiB
Dart
Executable file
360 lines
14 KiB
Dart
Executable file
#!/usr/bin/env dart
|
|
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
// Run tests like on the given builder.
|
|
|
|
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:args/args.dart';
|
|
|
|
import 'bots/results.dart';
|
|
|
|
const int deflakingCount = 5;
|
|
|
|
/// Quotes a string in shell single quote mode. This function produces a single
|
|
/// shell argument that evaluates to the exact string provided, handling any
|
|
/// special characters in the input string. Shell single quote mode works uses
|
|
/// the single quote character as the delimiter and uses the characters
|
|
/// in-between verbatim without any special processing. To insert the single
|
|
/// quote character itself, escape single quote mode, insert an escaped single
|
|
/// quote, and then return to single quote mode.
|
|
///
|
|
/// Examples:
|
|
/// foo becomes 'foo'
|
|
/// foo bar becomes 'foo bar'
|
|
/// foo\ bar becomes 'foo\ bar'
|
|
/// foo's bar becomes 'foo '\''s bar'
|
|
/// foo "b"ar becomes 'foo "b"'
|
|
/// foo
|
|
/// bar becomes 'foo
|
|
/// bar'
|
|
String shellSingleQuote(String string) {
|
|
return "'${string.replaceAll("'", "'\\''")}'";
|
|
}
|
|
|
|
/// Like [shellSingleQuote], but if the string only contains safe ASCII
|
|
/// characters, don't quote it. Note that it's not always safe to omit the
|
|
/// quotes even if the string only has safe characters, as doing so might match
|
|
/// a shell keyword or a shell builtin in the first argument in a command. It
|
|
/// should be safe to use this for the second argument onwards in a command.
|
|
String simpleShellSingleQuote(String string) {
|
|
return new RegExp(r"^[a-zA-Z0-9%+,./:_-]*$").hasMatch(string)
|
|
? string
|
|
: shellSingleQuote(string);
|
|
}
|
|
|
|
/// Runs a process and exits likewise if the process exits non-zero.
|
|
Future<ProcessResult> runProcess(
|
|
String executable, List<String> arguments) async {
|
|
final processResult = await Process.run(executable, arguments);
|
|
if (processResult.exitCode != 0) {
|
|
final command =
|
|
([executable]..addAll(arguments)).map(simpleShellSingleQuote).join(" ");
|
|
throw new Exception("Command exited ${processResult.exitCode}: $command\n"
|
|
"${processResult.stdout}\n${processResult.stderr}");
|
|
}
|
|
return processResult;
|
|
}
|
|
|
|
/// Runs a process and exits likewise if the process exits non-zero, but let the
|
|
/// child process inherit out stdio handles.
|
|
Future<ProcessResult> runProcessInheritStdio(
|
|
String executable, List<String> arguments) async {
|
|
final process = await Process.start(executable, arguments,
|
|
mode: ProcessStartMode.inheritStdio);
|
|
final exitCode = await process.exitCode;
|
|
final processResult = new ProcessResult(process.pid, exitCode, "", "");
|
|
if (processResult.exitCode != 0) {
|
|
final command =
|
|
([executable]..addAll(arguments)).map(simpleShellSingleQuote).join(" ");
|
|
throw new Exception("Command exited ${processResult.exitCode}: $command");
|
|
}
|
|
return processResult;
|
|
}
|
|
|
|
/// Returns the operating system of a builder.
|
|
String systemOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["linux", "mac", "win"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the product mode of a builder.
|
|
String modeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["debug", "product", "release"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the machine architecture of a builder.
|
|
String archOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => [
|
|
"arm",
|
|
"arm64",
|
|
"armsimdbc",
|
|
"armsimdbc64",
|
|
"ia32",
|
|
"simarm",
|
|
"simarm64",
|
|
"simdbc",
|
|
"simdbc64",
|
|
"x64",
|
|
].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the runtime environment of a builder.
|
|
String runtimeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["chrome", "d8", "edge", "firefox", "ie11", "safari"]
|
|
.contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Expands a variable in a test matrix step command.
|
|
String expandVariable(String string, String variable, String value) {
|
|
return string.replaceAll("\${$variable}", value ?? "");
|
|
}
|
|
|
|
/// Expands all variables in a test matrix step command.
|
|
String expandVariables(String string, String builder) {
|
|
string = expandVariable(string, "system", systemOfBuilder(builder));
|
|
string = expandVariable(string, "mode", modeOfBuilder(builder));
|
|
string = expandVariable(string, "arch", archOfBuilder(builder));
|
|
string = expandVariable(string, "runtime", runtimeOfBuilder(builder));
|
|
return string;
|
|
}
|
|
|
|
/// Locates the merge base between head and the [branch] on the given [remote].
|
|
/// If a particular [commit] was requested, use that.
|
|
Future<String> findMergeBase(
|
|
String commit, String remote, String branch) async {
|
|
if (commit != null) {
|
|
return commit;
|
|
}
|
|
final arguments = ["merge-base", "$remote/$branch", "HEAD"];
|
|
final result = await Process.run("git", arguments);
|
|
if (result.exitCode != 0) {
|
|
throw new Exception("Failed to run: git ${arguments.join(' ')}\n"
|
|
"stdout:\n${result.stdout}\n"
|
|
"stderr:\n${result.stderr}\n");
|
|
}
|
|
return LineSplitter.split(result.stdout).first;
|
|
}
|
|
|
|
/// Locates the build number of the [commit] on the [builder], or throws an
|
|
/// exception if the builder hasn't built the commit.
|
|
Future<int> buildNumberOfCommit(String builder, String commit) async {
|
|
final requestUrl = Uri.parse(
|
|
"https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search"
|
|
"?bucket=luci.dart.ci.sandbox"
|
|
"&tag=builder%3A$builder"
|
|
"&tag=buildset%3Acommit%2Fgit%2F$commit"
|
|
"&fields=builds(status%2Ctags%2Curl)");
|
|
final client = new HttpClient();
|
|
final request = await client.getUrl(requestUrl);
|
|
final response = await request.close();
|
|
final Map<String, dynamic> object = await response
|
|
.transform(new Utf8Decoder())
|
|
.transform(new JsonDecoder())
|
|
.first;
|
|
client.close();
|
|
final builds = object["builds"];
|
|
if (builds == null || builds.isEmpty) {
|
|
throw new Exception("Builder $builder hasn't built commit $commit");
|
|
}
|
|
final build = builds.last;
|
|
final tags = (build["tags"] as List).cast<String>();
|
|
final buildAddressTag =
|
|
tags.firstWhere((tag) => tag.startsWith("build_address:"));
|
|
final buildAddress = buildAddressTag.substring("build_address:".length);
|
|
if (build["status"] != "COMPLETED") {
|
|
throw new Exception("Build $buildAddress isn't completed yet");
|
|
}
|
|
return int.parse(buildAddress.split("/").last);
|
|
}
|
|
|
|
void main(List<String> args) async {
|
|
final parser = new ArgParser();
|
|
parser.addOption("builder",
|
|
abbr: "b", help: "Run tests like on the given buider");
|
|
parser.addOption("branch",
|
|
abbr: "B",
|
|
help: "Select the builders building this branch",
|
|
defaultsTo: "master");
|
|
parser.addOption("commit", abbr: "C", help: "Compare with this commit");
|
|
parser.addOption("remote",
|
|
abbr: "R",
|
|
help: "Compare with this remote and git branch",
|
|
defaultsTo: "origin");
|
|
parser.addFlag("help", help: "Show the program usage.", negatable: false);
|
|
|
|
final options = parser.parse(args);
|
|
if (options["help"] || options["builder"] == null) {
|
|
print("""
|
|
Usage: test.dart -b [BUILDER] [OPTION]...
|
|
Run tests and compare with the results on the given builder.
|
|
|
|
${parser.usage}""");
|
|
return;
|
|
}
|
|
|
|
final builder = options["builder"];
|
|
|
|
// Find out where the current HEAD branched.
|
|
final commit = await findMergeBase(
|
|
options["commit"], options["remote"], options["branch"]);
|
|
print("Base commit is $commit");
|
|
|
|
// Use the buildbucket API to search for builds of the right rcommit.
|
|
print("Finding build to compare with...");
|
|
final buildNumber = await buildNumberOfCommit(builder, commit);
|
|
print("Comparing with build $buildNumber on $builder");
|
|
|
|
final outDirectory = await Directory.systemTemp.createTemp("test.dart.");
|
|
try {
|
|
// Download the previous results and flakiness info from cloud storage.
|
|
print("Downloading previous results...");
|
|
await cpGsutil(
|
|
buildFileCloudPath(builder, buildNumber.toString(), "results.json"),
|
|
"${outDirectory.path}/previous.json");
|
|
await cpGsutil(
|
|
buildFileCloudPath(builder, buildNumber.toString(), "flaky.json"),
|
|
"${outDirectory.path}/flaky.json");
|
|
print("Downloaded previous results");
|
|
|
|
// Load the test matrix.
|
|
final scriptPath = Platform.script.toFilePath();
|
|
final testMatrixPath =
|
|
scriptPath.substring(0, scriptPath.length - "test.dart".length) +
|
|
"bots/test_matrix.json";
|
|
final testMatrix =
|
|
jsonDecode(await new File(testMatrixPath).readAsString());
|
|
|
|
// Find the appropriate test.py steps.
|
|
final buildersConfigurations = testMatrix["builder_configurations"];
|
|
final builderConfiguration = buildersConfigurations.firstWhere(
|
|
(builderConfiguration) =>
|
|
(builderConfiguration["builders"] as List).contains(builder));
|
|
final steps = (builderConfiguration["steps"] as List).cast<Map>();
|
|
final testSteps = steps
|
|
.where((step) =>
|
|
!step.containsKey("script") || step["script"] == "tools/test.py")
|
|
.toList();
|
|
|
|
// Run each step like the builder would, deflaking tests that need it.
|
|
final stepResultsPaths = <String>[];
|
|
final stepLogsPaths = <String>[];
|
|
for (int stepIndex = 0; stepIndex < testSteps.length; stepIndex++) {
|
|
// Run the test step.
|
|
final testStep = testSteps[stepIndex];
|
|
final stepName = testStep["name"];
|
|
final stepDirectory = new Directory("${outDirectory.path}/$stepIndex");
|
|
await stepDirectory.create();
|
|
final stepArguments = testStep["arguments"]
|
|
.map((argument) => expandVariables(argument, builder))
|
|
.toList()
|
|
.cast<String>();
|
|
final fullArguments = <String>[]
|
|
..addAll(stepArguments)
|
|
..addAll([
|
|
"--output-directory=${stepDirectory.path}",
|
|
"--clean-exit",
|
|
"--silent-failures",
|
|
"--write-results",
|
|
"--write-logs",
|
|
])
|
|
..addAll(options.rest);
|
|
print("".padLeft(80, "="));
|
|
print("$stepName: Running tests");
|
|
print("".padLeft(80, "="));
|
|
await runProcessInheritStdio("tools/test.py", fullArguments);
|
|
stepResultsPaths.add("${stepDirectory.path}/results.json");
|
|
stepLogsPaths.add("${stepDirectory.path}/logs.json");
|
|
// Find the list of tests to deflake.
|
|
final deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
|
|
"tools/bots/compare_results.dart",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${stepDirectory.path}/results.json",
|
|
]);
|
|
final deflakeListPath = "${stepDirectory.path}/deflake.list";
|
|
final deflakeListFile = new File(deflakeListPath);
|
|
await deflakeListFile.writeAsString(deflakeListOutput.stdout);
|
|
// Deflake the changed tests.
|
|
final deflakingResultsPaths = <String>[];
|
|
for (int i = 1;
|
|
deflakeListOutput.stdout != "" && i <= deflakingCount;
|
|
i++) {
|
|
print("".padLeft(80, "="));
|
|
print("$stepName: Running deflaking iteration $i");
|
|
print("".padLeft(80, "="));
|
|
final deflakeDirectory = new Directory("${stepDirectory.path}/$i");
|
|
await deflakeDirectory.create();
|
|
final deflakeArguments = <String>[]
|
|
..addAll(stepArguments)
|
|
..addAll([
|
|
"--output-directory=${deflakeDirectory.path}",
|
|
"--clean-exit",
|
|
"--silent-failures",
|
|
"--write-results",
|
|
"--test-list=$deflakeListPath",
|
|
])
|
|
..addAll(options.rest);
|
|
await runProcessInheritStdio("tools/test.py", deflakeArguments);
|
|
deflakingResultsPaths.add("${deflakeDirectory.path}/results.json");
|
|
}
|
|
// Update the flakiness information based on what we've learned.
|
|
print("$stepName: Updating flakiness information");
|
|
await runProcess(
|
|
Platform.resolvedExecutable,
|
|
[
|
|
"tools/bots/update_flakiness.dart",
|
|
"--input=${outDirectory.path}/flaky.json",
|
|
"--output=${outDirectory.path}/flaky.json",
|
|
"${stepDirectory.path}/results.json",
|
|
]..addAll(deflakingResultsPaths));
|
|
}
|
|
// Collect all the results from all the steps.
|
|
await new File("${outDirectory.path}/results.json").writeAsString(
|
|
stepResultsPaths
|
|
.map((path) => new File(path).readAsStringSync())
|
|
.join(""));
|
|
// Collect all the logs from all the steps.
|
|
await new File("${outDirectory.path}/logs.json").writeAsString(stepLogsPaths
|
|
.map((path) => new File(path).readAsStringSync())
|
|
.join(""));
|
|
// Write out the final comparison.
|
|
print("".padLeft(80, "="));
|
|
print("Test Results");
|
|
print("".padLeft(80, "="));
|
|
final compareOutput = await runProcess(Platform.resolvedExecutable, [
|
|
"tools/bots/compare_results.dart",
|
|
"--human",
|
|
"--verbose",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"--logs=${outDirectory.path}/logs.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${outDirectory.path}/results.json",
|
|
]);
|
|
if (compareOutput.stdout == "") {
|
|
print("There were no test failures.");
|
|
} else {
|
|
stdout.write(compareOutput.stdout);
|
|
}
|
|
} finally {
|
|
await outDirectory.delete(recursive: true);
|
|
}
|
|
}
|