mirror of
https://github.com/dart-lang/sdk
synced 2024-10-14 08:51:21 +00:00
927f0937bd
test.dart locates where the current branch branched off master and compares the local testing results with the appropriate mainline builder results, letting you know how the current change compares without the need for status files. Bug: https://github.com/dart-lang/sdk/issues/35086 Change-Id: Ib79479b867c5ac131302fea1bdf7effd0422a83a Reviewed-on: https://dart-review.googlesource.com/c/83281 Reviewed-by: Alexander Thomas <athom@google.com>
437 lines
15 KiB
Dart
Executable file
437 lines
15 KiB
Dart
Executable file
#!/usr/bin/env dart
|
|
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
/// List tests whose results are different from the previously approved results,
|
|
/// and ask whether to update the currently approved results, turning the bots
|
|
/// green.
|
|
|
|
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math';
|
|
|
|
import 'package:args/args.dart';
|
|
import 'package:glob/glob.dart';
|
|
|
|
import 'bots/results.dart';
|
|
|
|
/// The bot names and named configurations are highly redundant if both are
|
|
/// listed. This function returns a simplified named configuration that doesn't
|
|
/// contain any aspects that's part of the bots name. This is used to get a more
|
|
/// compact and readable output.
|
|
String simplifyNamedConfiguration(String bot, String namedConfiguration) {
|
|
final botComponents = new Set<String>.from(bot.split("-"));
|
|
return namedConfiguration
|
|
.split("-")
|
|
.where((component) => !botComponents.contains(component))
|
|
.join("-");
|
|
}
|
|
|
|
/// Represents a test on a bot with the current result, the current approved
|
|
/// result, and flakiness data.
|
|
class Test implements Comparable {
|
|
final String bot;
|
|
final String name;
|
|
final Map<String, dynamic> resultData;
|
|
final Map<String, dynamic> approvedResultData;
|
|
final Map<String, dynamic> flakinessData;
|
|
|
|
Test(this.bot, this.name, this.resultData, this.approvedResultData,
|
|
this.flakinessData);
|
|
|
|
int compareTo(Object other) {
|
|
if (other is Test) {
|
|
if (bot.compareTo(other.bot) < 0) return -1;
|
|
if (other.bot.compareTo(bot) < 0) return 1;
|
|
if (configuration.compareTo(other.configuration) < 0) return -1;
|
|
if (other.configuration.compareTo(configuration) < 0) return 1;
|
|
if (name.compareTo(other.name) < 0) return -1;
|
|
if (other.name.compareTo(name) < 0) return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
String get configuration => resultData["configuration"];
|
|
String get result => resultData["result"];
|
|
String get expected => resultData["expected"];
|
|
bool get matches => resultData["matches"];
|
|
String get approvedResult =>
|
|
approvedResultData != null ? approvedResultData["result"] : null;
|
|
bool get isApproved => result == approvedResult;
|
|
List<String> get flakyModes =>
|
|
flakinessData != null ? flakinessData["outcomes"].cast<String>() : null;
|
|
bool get isFlake => flakinessData != null && flakyModes.contains(result);
|
|
}
|
|
|
|
/// Loads the results file as as a map if the file exists, otherwise returns the
|
|
/// empty map.
|
|
Future<Map<String, Map<String, dynamic>>> loadResultsMapIfExists(
|
|
String path) async =>
|
|
await new File(path).exists()
|
|
? loadResultsMap(path)
|
|
: <String, Map<String, dynamic>>{};
|
|
|
|
/// Loads the results from the bot.
|
|
Future<List<Test>> loadResultsFromBot(String bot, ArgResults options) async {
|
|
if (options["verbose"]) {
|
|
print("Loading $bot...");
|
|
}
|
|
// gsutil cp -r requires a destination directory, use a temporary one.
|
|
final tmpdir = await Directory.systemTemp.createTemp("approve_results.");
|
|
try {
|
|
// The 'latest' file contains the name of the latest build that we
|
|
// should download.
|
|
final build = await readFile(bot, "latest");
|
|
|
|
// Asynchronously download the latest build and the current approved
|
|
// results.
|
|
await Future.wait([
|
|
cpRecursiveGsutil(buildCloudPath(bot, build), tmpdir.path),
|
|
cpRecursiveGsutil(
|
|
"$approvedResultsStoragePath/$bot/approved_results.json",
|
|
"${tmpdir.path}/approved_results.json"),
|
|
]);
|
|
|
|
// Check the build was properly downloaded.
|
|
final buildPath = "${tmpdir.path}/$build";
|
|
final buildDirectory = new Directory(buildPath);
|
|
if (!await buildDirectory.exists()) {
|
|
print("$bot: Build directory didn't exist");
|
|
return <Test>[];
|
|
}
|
|
|
|
// Load the run.json to find the named configuration.
|
|
final resultsFile = new File("$buildPath/results.json");
|
|
if (!await resultsFile.exists()) {
|
|
print("$bot: No results.json exists");
|
|
return <Test>[];
|
|
}
|
|
|
|
// Load the current results, the approved resutls, and the flakiness
|
|
// information.
|
|
final results = await loadResultsMapIfExists("$buildPath/results.json");
|
|
final flaky = await loadResultsMapIfExists("$buildPath/flaky.json");
|
|
final approvedResults =
|
|
await loadResultsMapIfExists("${tmpdir.path}/approved_results.json");
|
|
|
|
// Construct an object for every test containing its current result,
|
|
// what the last approved result was, and whether it's flaky.
|
|
final tests = <Test>[];
|
|
for (final key in results.keys) {
|
|
final result = results[key];
|
|
final approvedResult = approvedResults[key];
|
|
final flakiness = flaky[key];
|
|
final name = result["name"];
|
|
final test = new Test(bot, name, result, approvedResult, flakiness);
|
|
tests.add(test);
|
|
}
|
|
if (options["verbose"]) {
|
|
print("Loaded $bot (${tests.length} tests).");
|
|
}
|
|
return tests;
|
|
} finally {
|
|
// Always clean up the temporary directory when we don't need it.
|
|
await tmpdir.delete(recursive: true);
|
|
}
|
|
}
|
|
|
|
main(List<String> args) async {
|
|
final parser = new ArgParser();
|
|
parser.addMultiOption("bot",
|
|
abbr: "b",
|
|
help: "Select the bots matching the glob pattern [option is repeatable]",
|
|
splitCommas: false);
|
|
parser.addFlag("help", help: "Show the program usage.", negatable: false);
|
|
parser.addFlag("list",
|
|
abbr: "l", help: "List the available bots.", negatable: false);
|
|
parser.addFlag("no",
|
|
abbr: "n",
|
|
help: "Show changed results but don't approve.",
|
|
negatable: false);
|
|
parser.addFlag("verbose",
|
|
abbr: "v", help: "Describe asynchronous operations.", negatable: false);
|
|
parser.addFlag("yes",
|
|
abbr: "y", help: "Approve the results.", negatable: false);
|
|
|
|
final options = parser.parse(args);
|
|
if ((options["bot"].isEmpty && !options["list"]) || options["help"]) {
|
|
print("""
|
|
Usage: approve_results.dart [OPTION]...
|
|
List tests whose results are different from the previously approved results, and
|
|
ask whether to update the currently approved results, turning the bots green.
|
|
|
|
The options are as follows:
|
|
|
|
${parser.usage}""");
|
|
return;
|
|
}
|
|
|
|
if (options["no"] && options["yes"]) {
|
|
stderr.writeln("The --no and --yes options are mutually incompatible");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
// Load the list of bots according to the test matrix.
|
|
final testMatrixPath =
|
|
Platform.script.resolve("bots/test_matrix.json").toFilePath();
|
|
final testMatrix = jsonDecode(await new File(testMatrixPath).readAsString());
|
|
final builderConfigurations = testMatrix["builder_configurations"];
|
|
final testMatrixBots = <String>[];
|
|
for (final builderConfiguration in builderConfigurations) {
|
|
final steps = builderConfiguration["steps"];
|
|
// Only consider bots that use tools/test.py.
|
|
if (!steps.any((step) =>
|
|
step["script"] == null || step["script"] == "tools/test.py")) {
|
|
continue;
|
|
}
|
|
final builders = builderConfiguration["builders"].cast<String>();
|
|
testMatrixBots.addAll(builders);
|
|
}
|
|
|
|
// Load the list of bots that have data in cloud storage.
|
|
if (options["verbose"]) {
|
|
print("Loading list of bots...");
|
|
}
|
|
final botsWithData = (await listBots())
|
|
.where((bot) => !bot.endsWith("-try"))
|
|
.where((bot) => !bot.endsWith("-dev"))
|
|
.where((bot) => !bot.endsWith("-stable"));
|
|
if (options["verbose"]) {
|
|
print("Loaded list of bots.");
|
|
}
|
|
|
|
// The currently active bots are the bots both mentioned in the test matrix
|
|
// and that have results in cloud storage.
|
|
final allBots = new Set<String>.from(testMatrixBots)
|
|
.intersection(new Set<String>.from(botsWithData))
|
|
.toList()
|
|
..sort();
|
|
|
|
// List the currently active bots if requested.
|
|
if (options["list"]) {
|
|
for (final bot in allBots) {
|
|
print(bot);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Select all the bots matching the glob patterns,
|
|
final bots = new Set<String>();
|
|
for (final botPattern in options["bot"]) {
|
|
final glob = new Glob(botPattern);
|
|
bool any = false;
|
|
for (final bot in allBots) {
|
|
if (glob.matches(bot)) {
|
|
bots.add(bot);
|
|
any = true;
|
|
}
|
|
}
|
|
if (!any) {
|
|
stderr.writeln("error: No bots matched pattern: $botPattern");
|
|
stderr.writeln("Try --list to get the list of bots, or --help for help");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
}
|
|
for (final bot in bots) {
|
|
print("Selected bot: $bot");
|
|
}
|
|
|
|
// Load all the latest results for the selected bots, as well as flakiness
|
|
// data, and the set of currently approved results. Each bot's latest build
|
|
// is downloaded in parallel to make this phase faster.
|
|
final testListFutures = <Future>[];
|
|
for (final String bot in bots) {
|
|
testListFutures.add(loadResultsFromBot(bot, options));
|
|
}
|
|
|
|
// Collect all the tests from the synchronous downloads.
|
|
final tests = <Test>[];
|
|
for (final testList in await Future.wait(testListFutures)) {
|
|
tests.addAll(testList);
|
|
}
|
|
tests.sort();
|
|
print("");
|
|
|
|
// Compute statistics and the set of interesting tests.
|
|
final flakyTestsCount = tests.where((test) => test.isFlake).length;
|
|
final failingTestsCount =
|
|
tests.where((test) => !test.isFlake && !test.matches).length;
|
|
final unapprovedTests =
|
|
tests.where((test) => !test.isFlake && !test.isApproved).toList();
|
|
final fixedTests = unapprovedTests.where((test) => test.matches).toList();
|
|
final brokenTests = unapprovedTests.where((test) => !test.matches).toList();
|
|
|
|
// Find out which bots have multiple configurations.
|
|
final outcomes = new Set<String>();
|
|
final configurationsForBots = <String, Set<String>>{};
|
|
for (final test in tests) {
|
|
outcomes.add(test.result);
|
|
var configurationSet = configurationsForBots[test.bot];
|
|
if (configurationSet == null) {
|
|
configurationsForBots[test.bot] = configurationSet = new Set<String>();
|
|
}
|
|
configurationSet.add(test.configuration);
|
|
}
|
|
|
|
// Compute a nice displayed name for the bot and configuration. If the bot
|
|
// only has a single configuration, then only mention the bot. Otherwise,
|
|
// remove the redundant parts from configuration and present it compactly.
|
|
// This is needed to avoid the tables becoming way too large.
|
|
String getBotDisplayName(String bot, String configuration) {
|
|
if (configurationsForBots[bot].length == 1) {
|
|
return bot;
|
|
} else {
|
|
final simpleConfig = simplifyNamedConfiguration(bot, configuration);
|
|
return "$bot/$simpleConfig";
|
|
}
|
|
}
|
|
|
|
// Compute the width of the fields in the below tables.
|
|
final unapprovedBots = new Set<String>();
|
|
int longestBot = "BOT/CONFIG".length;
|
|
int longestTest = "TEST".length;
|
|
int longestResult = "RESULT".length;
|
|
for (final test in unapprovedTests) {
|
|
unapprovedBots.add(test.bot);
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
longestBot = max(longestBot, botDisplayName.length);
|
|
if (!test.matches) {
|
|
longestTest = max(longestTest, test.name.length);
|
|
longestResult = max(longestResult, test.result.length);
|
|
}
|
|
}
|
|
longestTest = min(longestTest, 120); // Some tests names are extremely long.
|
|
|
|
// Table of lists that now succeed.
|
|
if (fixedTests.isNotEmpty) {
|
|
print("The following tests are now succeeding:\n");
|
|
print("${'BOT/CONFIG'.padRight(longestBot)} "
|
|
"TEST");
|
|
for (final test in fixedTests) {
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
print("${botDisplayName.padRight(longestBot)} "
|
|
"${test.name}");
|
|
}
|
|
print("");
|
|
}
|
|
|
|
/// Table of lists that now fail.
|
|
if (brokenTests.isNotEmpty) {
|
|
print("The following tests are now failing:\n");
|
|
print("${'BOT'.padRight(longestBot)} "
|
|
"${'TEST'.padRight(longestTest)} "
|
|
"${'RESULT'.padRight(longestResult)} "
|
|
"EXPECTED");
|
|
for (final test in brokenTests) {
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
print("${botDisplayName.padRight(longestBot)} "
|
|
"${test.name.padRight(longestTest)} "
|
|
"${test.result.padRight(longestResult)} "
|
|
"${test.expected}");
|
|
}
|
|
print("");
|
|
}
|
|
|
|
// Provide statistics on how well the bots are doing.
|
|
void statistic(int numerator, int denominator, String what) {
|
|
double percent = numerator / denominator * 100.0;
|
|
String percentString = percent.toStringAsFixed(2) + "%";
|
|
print("$numerator of $denominator $what ($percentString)");
|
|
}
|
|
|
|
statistic(failingTestsCount, tests.length, "tests are failing");
|
|
statistic(flakyTestsCount, tests.length, "tests are flaky");
|
|
statistic(
|
|
fixedTests.length, tests.length, "tests were fixed since last approval");
|
|
statistic(brokenTests.length, tests.length,
|
|
"tests were broken since last approval");
|
|
|
|
// Stop if there's nothing to do.
|
|
if (unapprovedBots.isEmpty) {
|
|
print("\nEvery test result has already been approved.");
|
|
return;
|
|
}
|
|
|
|
// Stop if this is a dry run.
|
|
if (options["no"]) {
|
|
if (unapprovedBots.length == 1) {
|
|
print("1 test has a changed result and needs approval");
|
|
} else {
|
|
print("${unapprovedBots.length} "
|
|
"tests have changed results and need approval");
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Confirm the approval if run interactively.
|
|
if (!options["yes"]) {
|
|
print("");
|
|
print("Note: It is assumed bugs have been filed about the above failures "
|
|
"before they are approved here.");
|
|
if (brokenTests.isNotEmpty) {
|
|
final botPlural = bots.length == 1 ? "bot" : "bots";
|
|
print("Note: Approving the failures will turn the "
|
|
"$botPlural green on the next commit.");
|
|
}
|
|
while (true) {
|
|
stdout.write("Do you want to approve? (yes/no) [yes] ");
|
|
final line = stdin.readLineSync();
|
|
// End of file condition is considered no.
|
|
if (line == null) {
|
|
print("n");
|
|
return;
|
|
}
|
|
if (line.toLowerCase() == "n" || line.toLowerCase() == "no") {
|
|
return;
|
|
}
|
|
if (line == "" ||
|
|
line.toLowerCase() == "y" ||
|
|
line.toLowerCase() == "yes") {
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
print("Note: It is assumed bugs have been filed about the above failures.");
|
|
}
|
|
print("");
|
|
|
|
// Update approved_results.json for each bot with unapproved changes.
|
|
final outDirectory =
|
|
await Directory.systemTemp.createTemp("approved_results.");
|
|
try {
|
|
final testsForBots = <String, List<Test>>{};
|
|
for (final test in tests) {
|
|
if (!testsForBots.containsKey(test.bot)) {
|
|
testsForBots[test.bot] = <Test>[test];
|
|
} else {
|
|
testsForBots[test.bot].add(test);
|
|
}
|
|
}
|
|
print("Uploading approved results...");
|
|
final futures = <Future>[];
|
|
for (final String bot in unapprovedBots) {
|
|
final testsList = testsForBots[bot];
|
|
final localPath = "${outDirectory.path}/$bot.json";
|
|
await new File(localPath).writeAsString(
|
|
testsList.map((test) => jsonEncode(test.resultData) + "\n").join(""));
|
|
final remotePath =
|
|
"$approvedResultsStoragePath/$bot/approved_results.json";
|
|
futures.add(cpGsutil(localPath, remotePath)
|
|
.then((_) => print("Uploaded approved results for $bot")));
|
|
}
|
|
await Future.wait(futures);
|
|
if (brokenTests.isNotEmpty) {
|
|
print(
|
|
"Successfully approved results, the next commit will turn bots green");
|
|
} else {
|
|
print("Successfully approved results");
|
|
}
|
|
} finally {
|
|
await outDirectory.delete(recursive: true);
|
|
}
|
|
}
|