mirror of
https://github.com/dart-lang/sdk
synced 2024-10-14 09:01:42 +00:00
927f0937bd
test.dart locates where the current branch branched off master and compares the local testing results with the appropriate mainline builder results, letting you know how the current change compares without the need for status files. Bug: https://github.com/dart-lang/sdk/issues/35086 Change-Id: Ib79479b867c5ac131302fea1bdf7effd0422a83a Reviewed-on: https://dart-review.googlesource.com/c/83281 Reviewed-by: Alexander Thomas <athom@google.com>
276 lines
10 KiB
Dart
Executable file
276 lines
10 KiB
Dart
Executable file
#!/usr/bin/env dart
|
|
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
// Run tests like on the given builder.
|
|
|
|
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:args/args.dart';
|
|
|
|
import 'bots/results.dart';
|
|
|
|
const int deflakingCount = 5;
|
|
|
|
/// Returns the operating system of a builder.
|
|
String systemOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["linux", "mac", "win"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the product mode of a builder.
|
|
String modeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["debug", "product", "release"].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the machine architecture of a builder.
|
|
String archOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => [
|
|
"arm",
|
|
"arm64",
|
|
"armsimdbc",
|
|
"armsimdbc64",
|
|
"ia32",
|
|
"simarm",
|
|
"simarm64",
|
|
"simdbc",
|
|
"simdbc64",
|
|
"x64",
|
|
].contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Returns the runtime environment of a builder.
|
|
String runtimeOfBuilder(String builder) {
|
|
return builder.split("-").firstWhere(
|
|
(component) => ["chrome", "d8", "edge", "firefox", "ie11", "safari"]
|
|
.contains(component),
|
|
orElse: () => null);
|
|
}
|
|
|
|
/// Expands a variable in a test matrix step command.
|
|
String expandVariable(String string, String variable, String value) {
|
|
return string.replaceAll("\${$variable}", value ?? "");
|
|
}
|
|
|
|
/// Expands all variables in a test matrix step command.
|
|
String expandVariables(String string, String builder) {
|
|
string = expandVariable(string, "system", systemOfBuilder(builder));
|
|
string = expandVariable(string, "mode", modeOfBuilder(builder));
|
|
string = expandVariable(string, "arch", archOfBuilder(builder));
|
|
string = expandVariable(string, "runtime", runtimeOfBuilder(builder));
|
|
return string;
|
|
}
|
|
|
|
/// Locates the merge base between head and the [branch] on the given [remote].
|
|
/// If a particular [commit] was requested, use that.
|
|
Future<String> findMergeBase(
|
|
String commit, String remote, String branch) async {
|
|
if (commit != null) {
|
|
return commit;
|
|
}
|
|
final arguments = ["merge-base", "$remote/$branch", "HEAD"];
|
|
final result = await Process.run("git", arguments);
|
|
if (result.exitCode != 0) {
|
|
throw new Exception("Failed to run: git ${arguments.join(' ')}\n"
|
|
"stdout:\n${result.stdout}\n"
|
|
"stderr:\n${result.stderr}\n");
|
|
}
|
|
return LineSplitter.split(result.stdout).first;
|
|
}
|
|
|
|
/// Locates the build number of the [commit] on the [builder], or throws an
|
|
/// exception if the builder hasn't built the commit.
|
|
Future<int> buildNumberOfCommit(String builder, String commit) async {
|
|
final requestUrl = Uri.parse(
|
|
"https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search"
|
|
"?bucket=luci.dart.ci.sandbox"
|
|
"&tag=builder%3A$builder"
|
|
"&tag=buildset%3Acommit%2Fgit%2F$commit"
|
|
"&fields=builds(status%2Ctags%2Curl)");
|
|
final client = new HttpClient();
|
|
final request = await client.getUrl(requestUrl);
|
|
final response = await request.close();
|
|
final Map<String, dynamic> object = await response
|
|
.transform(new Utf8Decoder())
|
|
.transform(new JsonDecoder())
|
|
.first;
|
|
client.close();
|
|
final builds = object["builds"];
|
|
if (builds.isEmpty) {
|
|
throw new Exception("Builder $builder hasn't built commit $commit");
|
|
}
|
|
final build = builds.last;
|
|
final tags = (build["tags"] as List).cast<String>();
|
|
final buildAddressTag =
|
|
tags.firstWhere((tag) => tag.startsWith("build_address:"));
|
|
final buildAddress = buildAddressTag.substring("build_address:".length);
|
|
if (build["status"] != "COMPLETED") {
|
|
throw new Exception("Build $buildAddress isn't completed yet");
|
|
}
|
|
return int.parse(buildAddress.split("/").last);
|
|
}
|
|
|
|
void main(List<String> args) async {
|
|
final parser = new ArgParser();
|
|
parser.addOption("builder",
|
|
abbr: "b", help: "Run tests like on the given buider");
|
|
parser.addOption("branch",
|
|
abbr: "B",
|
|
help: "Select the builders building this branch",
|
|
defaultsTo: "master");
|
|
parser.addOption("commit", abbr: "C", help: "Compare with this commit");
|
|
parser.addOption("remote",
|
|
abbr: "R",
|
|
help: "Compare with this remote and git branch",
|
|
defaultsTo: "origin");
|
|
parser.addFlag("help", help: "Show the program usage.", negatable: false);
|
|
|
|
final options = parser.parse(args);
|
|
if (options["help"] || options["builder"] == null) {
|
|
print("""
|
|
Usage: test.dart -b [BUILDER] [OPTION]...
|
|
Run tests and compare with the results on the given builder.
|
|
|
|
${parser.usage}""");
|
|
return;
|
|
}
|
|
|
|
final builder = options["builder"];
|
|
|
|
// Find out where the current HEAD branched.
|
|
final commit = await findMergeBase(
|
|
options["commit"], options["remote"], options["branch"]);
|
|
print("Base commit is $commit");
|
|
|
|
// Use the buildbucket API to search for builds of the right rcommit.
|
|
print("Finding build to compare with...");
|
|
final buildNumber = await buildNumberOfCommit(builder, commit);
|
|
print("Comparing with build $buildNumber on $builder");
|
|
|
|
final outDirectory = await Directory.systemTemp.createTemp("test.dart.");
|
|
try {
|
|
// Download the previous results and flakiness info from cloud storage.
|
|
print("Downloading previous results...");
|
|
await cpGsutil(
|
|
buildFileCloudPath(builder, buildNumber.toString(), "results.json"),
|
|
"${outDirectory.path}/previous.json");
|
|
await cpGsutil(
|
|
buildFileCloudPath(builder, buildNumber.toString(), "flaky.json"),
|
|
"${outDirectory.path}/flaky.json");
|
|
print("Downloaded previous results");
|
|
|
|
// Load the test matrix.
|
|
final scriptPath = Platform.script.toFilePath();
|
|
final testMatrixPath =
|
|
scriptPath.substring(0, scriptPath.length - "test.dart".length) +
|
|
"bots/test_matrix.json";
|
|
final testMatrix =
|
|
jsonDecode(await new File(testMatrixPath).readAsString());
|
|
|
|
// Find the appropriate test.py steps.
|
|
final buildersConfigurations = testMatrix["builder_configurations"];
|
|
final builderConfiguration = buildersConfigurations.firstWhere(
|
|
(builderConfiguration) =>
|
|
(builderConfiguration["builders"] as List).contains(builder));
|
|
final steps = (builderConfiguration["steps"] as List).cast<Map>();
|
|
final testSteps = steps
|
|
.where((step) =>
|
|
!step.containsKey("script") || step["script"] == "tools/test.py")
|
|
.toList();
|
|
|
|
// Run each step like the builder would, deflaking tests that need it.
|
|
final stepResultsPaths = <String>[];
|
|
for (int stepIndex = 0; stepIndex < testSteps.length; stepIndex++) {
|
|
// Run the test step.
|
|
final testStep = testSteps[stepIndex];
|
|
final stepName = testStep["name"];
|
|
final stepDirectory = new Directory("${outDirectory.path}/$stepIndex");
|
|
await stepDirectory.create();
|
|
final stepArguments = testStep["arguments"]
|
|
.map((argument) => expandVariables(argument, builder))
|
|
.toList()
|
|
.cast<String>();
|
|
final fullArguments = <String>[]
|
|
..addAll(stepArguments)
|
|
..addAll(
|
|
["--output-directory=${stepDirectory.path}", "--write-results"])
|
|
..addAll(options.rest);
|
|
print("$stepName: Running tests");
|
|
final testProcess = await Process.start("tools/test.py", fullArguments);
|
|
await testProcess.exitCode;
|
|
stepResultsPaths.add("${stepDirectory.path}/results.json");
|
|
// Find the list of tests to deflake.
|
|
final deflakeListOutput = await Process.run(Platform.resolvedExecutable, [
|
|
"tools/bots/compare_results.dart",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${stepDirectory.path}/results.json"
|
|
]);
|
|
final deflakeListPath = "${stepDirectory.path}/deflake.list";
|
|
final deflakeListFile = new File(deflakeListPath);
|
|
await deflakeListFile.writeAsString(deflakeListOutput.stdout);
|
|
// Deflake the changed tests.
|
|
final deflakingResultsPaths = <String>[];
|
|
for (int i = 1;
|
|
deflakeListOutput.stdout != "" && i <= deflakingCount;
|
|
i++) {
|
|
print("$stepName: Running deflaking iteration $i");
|
|
final deflakeDirectory = new Directory("${stepDirectory.path}/$i");
|
|
await deflakeDirectory.create();
|
|
final deflakeArguments = <String>[]
|
|
..addAll(stepArguments)
|
|
..addAll([
|
|
"--output-directory=${deflakeDirectory.path}",
|
|
"--write-results",
|
|
"--test-list=$deflakeListPath"
|
|
])
|
|
..addAll(options.rest);
|
|
final deflakeProcess =
|
|
await Process.start("tools/test.py", deflakeArguments);
|
|
await deflakeProcess.exitCode;
|
|
deflakingResultsPaths.add("${deflakeDirectory.path}/results.json");
|
|
}
|
|
// Update the flakiness information based on what we've learned.
|
|
print("$stepName: Updating flakiness information");
|
|
await Process.run(
|
|
"tools/bots/update_flakiness.dart",
|
|
[
|
|
"--input=${outDirectory.path}/flaky.json",
|
|
"--output=${outDirectory.path}/flaky.json",
|
|
"${stepDirectory.path}/results.json",
|
|
]..addAll(deflakingResultsPaths));
|
|
}
|
|
// Collect all the results from all the steps.
|
|
await new File("${outDirectory.path}/results.json").writeAsString(
|
|
stepResultsPaths
|
|
.map((path) => new File(path).readAsStringSync())
|
|
.join(""));
|
|
// Write out the final comparison.
|
|
print("");
|
|
final compareOutput = await Process.run("tools/bots/compare_results.dart", [
|
|
"--human",
|
|
"--verbose",
|
|
"--changed",
|
|
"--failing",
|
|
"--passing",
|
|
"--flakiness-data=${outDirectory.path}/flaky.json",
|
|
"${outDirectory.path}/previous.json",
|
|
"${outDirectory.path}/results.json"
|
|
]);
|
|
stdout.write(compareOutput.stdout);
|
|
} finally {
|
|
await outDirectory.delete(recursive: true);
|
|
}
|
|
}
|