mirror of
https://github.com/dart-lang/sdk
synced 2024-11-05 18:22:09 +00:00
58470142b7
TBR=whesse@google.com Change-Id: Iceea303edb157d8653021ea55d746ba8a93124e5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/110908 Reviewed-by: Alexander Thomas <athom@google.com>
1006 lines
38 KiB
Dart
Executable file
1006 lines
38 KiB
Dart
Executable file
#!/usr/bin/env dart
|
|
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
/// List tests whose results are different from the previously approved results,
|
|
/// and ask whether to update the currently approved results, turning the bots
|
|
/// green.
|
|
|
|
import 'dart:async';
|
|
import 'dart:collection';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math';
|
|
|
|
import 'package:args/args.dart';
|
|
import 'package:glob/glob.dart';
|
|
|
|
import 'bots/results.dart';
|
|
|
|
/// Returns whether two decoded JSON objects are identical.
|
|
bool isIdenticalJson(dynamic a, dynamic b) {
|
|
if (a is Map<String, dynamic> && b is Map<String, dynamic>) {
|
|
if (a.length != b.length) return false;
|
|
for (final key in a.keys) {
|
|
if (!b.containsKey(key)) return false;
|
|
if (!isIdenticalJson(a[key], b[key])) return false;
|
|
}
|
|
return true;
|
|
} else if (a is List<dynamic> && b is List<dynamic>) {
|
|
if (a.length != b.length) return false;
|
|
for (int i = 0; i < a.length; i++) {
|
|
if (!isIdenticalJson(a[i], b[i])) return false;
|
|
}
|
|
return true;
|
|
} else {
|
|
return a == b;
|
|
}
|
|
}
|
|
|
|
/// Returns whether two sets of approvals are identical.
|
|
bool isIdenticalApprovals(
|
|
Map<String, Map<String, dynamic>> a, Map<String, Map<String, dynamic>> b) {
|
|
if (a.length != b.length) return false;
|
|
for (final key in a.keys) {
|
|
if (!b.containsKey(key)) return false;
|
|
if (!isIdenticalJson(a[key], b[key])) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
/// The bot names and named configurations are highly redundant if both are
|
|
/// listed. This function returns a simplified named configuration that doesn't
|
|
/// contain any aspects that's part of the bots name. This is used to get a more
|
|
/// compact and readable output.
|
|
String simplifyNamedConfiguration(String bot, String namedConfiguration) {
|
|
final botComponents = new Set<String>.from(bot.split("-"));
|
|
return namedConfiguration
|
|
.split("-")
|
|
.where((component) => !botComponents.contains(component))
|
|
.join("-");
|
|
}
|
|
|
|
/// Represents a test on a bot with the baseline results (if tryrun), the
|
|
/// current result, the current approved result, and flakiness data.
|
|
class Test implements Comparable {
|
|
final String bot;
|
|
final Map<String, dynamic> baselineData;
|
|
final Map<String, dynamic> resultData;
|
|
final Map<String, dynamic> approvedResultData;
|
|
final Map<String, dynamic> flakinessData;
|
|
|
|
Test(this.bot, this.baselineData, this.resultData, this.approvedResultData,
|
|
this.flakinessData);
|
|
|
|
int compareTo(Object other) {
|
|
if (other is Test) {
|
|
if (bot.compareTo(other.bot) < 0) return -1;
|
|
if (other.bot.compareTo(bot) < 0) return 1;
|
|
if (configuration.compareTo(other.configuration) < 0) return -1;
|
|
if (other.configuration.compareTo(configuration) < 0) return 1;
|
|
if (name.compareTo(other.name) < 0) return -1;
|
|
if (other.name.compareTo(name) < 0) return 1;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
Map<String, dynamic> get _sharedData =>
|
|
resultData ?? baselineData ?? approvedResultData;
|
|
String get name => _sharedData["name"];
|
|
String get configuration => _sharedData["configuration"];
|
|
String get key => "$configuration:$name";
|
|
String get expected => _sharedData["expected"];
|
|
String get result => (resultData ?? const {})["result"];
|
|
bool get matches => _sharedData["matches"];
|
|
String get baselineResult => (baselineData ?? const {})["result"];
|
|
String get approvedResult => (approvedResultData ?? const {})["result"];
|
|
bool get isDifferent => result != null && result != baselineResult;
|
|
bool get isApproved => result == null || result == approvedResult;
|
|
List<String> get flakyModes =>
|
|
flakinessData != null ? flakinessData["outcomes"].cast<String>() : null;
|
|
bool get isFlake => flakinessData != null && flakyModes.contains(result);
|
|
}
|
|
|
|
/// Loads the results file as as a map if the file exists, otherwise returns the
|
|
/// empty map.
|
|
Future<Map<String, Map<String, dynamic>>> loadResultsMapIfExists(
|
|
String path) async =>
|
|
await new File(path).exists()
|
|
? loadResultsMap(path)
|
|
: <String, Map<String, dynamic>>{};
|
|
|
|
/// Exception for when the results for a builder can't be found.
|
|
class NoResultsException implements Exception {
|
|
final String message;
|
|
final String buildUrl;
|
|
|
|
NoResultsException(this.message, this.buildUrl);
|
|
|
|
String toString() => message;
|
|
}
|
|
|
|
/// Loads a log from logdog.
|
|
Future<String> loadLog(String id, String step) async {
|
|
final buildUrl = "https://ci.chromium.org/b/$id";
|
|
final logUrl = Uri.parse("https://logs.chromium.org/"
|
|
"logs/dart/buildbucket/cr-buildbucket.appspot.com/"
|
|
"$id/+/steps/$step?format=raw");
|
|
final client = new HttpClient();
|
|
try {
|
|
final request =
|
|
await client.getUrl(logUrl).timeout(const Duration(seconds: 60));
|
|
final response = await request.close().timeout(const Duration(seconds: 60));
|
|
if (response.statusCode == HttpStatus.notFound) {
|
|
await response.drain();
|
|
throw new NoResultsException(
|
|
"The log at $logUrl doesn't exist: ${response.statusCode}", buildUrl);
|
|
}
|
|
if (response.statusCode != HttpStatus.ok) {
|
|
await response.drain();
|
|
throw new Exception("Failed to download $logUrl: ${response.statusCode}");
|
|
}
|
|
final contents = (await response
|
|
.cast<List<int>>()
|
|
.transform(new Utf8Decoder())
|
|
.timeout(const Duration(seconds: 60))
|
|
.toList())
|
|
.join("");
|
|
return contents;
|
|
} finally {
|
|
client.close();
|
|
}
|
|
}
|
|
|
|
/// TODO(https://github.com/dart-lang/sdk/issues/36015): The step name changed
|
|
/// incompatibly, allow both temporarily to reduce the user breakage. Remove
|
|
/// this 2019-03-25.
|
|
Future<String> todoFallbackLoadLog(
|
|
String id, String primary, String secondary) async {
|
|
try {
|
|
return await loadLog(id, primary);
|
|
} catch (e) {
|
|
if (e.toString().startsWith("Exception: The log at ") &&
|
|
e.toString().endsWith(" doesn't exist")) {
|
|
return await loadLog(id, secondary);
|
|
}
|
|
rethrow;
|
|
}
|
|
}
|
|
|
|
/// Loads the results from the bot.
|
|
Future<List<Test>> loadResultsFromBot(String bot, ArgResults options,
|
|
String changeId, Map<String, dynamic> changelistBuild) async {
|
|
if (options["verbose"]) {
|
|
print("Loading $bot...");
|
|
}
|
|
// gsutil cp -r requires a destination directory, use a temporary one.
|
|
final tmpdir = await Directory.systemTemp.createTemp("approve_results.");
|
|
try {
|
|
// The 'latest' file contains the name of the latest build that we
|
|
// should download. When preapproving a changelist, we instead find out
|
|
// which build the commit queue was rebased on.
|
|
/// TODO(https://github.com/dart-lang/sdk/issues/36015): The step name
|
|
/// changed incompatibly, allow both temporarily to reduce the user
|
|
/// breakage. Remove this 2019-03-25.
|
|
final build = (changeId != null
|
|
? await todoFallbackLoadLog(
|
|
changelistBuild["id"],
|
|
"download_previous_results/0/steps/gsutil_find_latest_build/0/logs/"
|
|
"raw_io.output_text_latest_/0",
|
|
"gsutil_find_latest_build/0/logs/raw_io.output_text_latest_/0")
|
|
: await readFile(bot, "latest"))
|
|
.trim();
|
|
|
|
// Asynchronously download the latest build and the current approved
|
|
// results. Download try results from trybot try runs if preapproving.
|
|
final tryResults = <String, Map<String, dynamic>>{};
|
|
await Future.wait([
|
|
cpRecursiveGsutil(buildCloudPath(bot, build), tmpdir.path),
|
|
cpRecursiveGsutil(
|
|
"$approvedResultsStoragePath/$bot/approved_results.json",
|
|
"${tmpdir.path}/approved_results.json"),
|
|
new Future(() async {
|
|
if (changeId != null) {
|
|
tryResults.addAll(parseResultsMap(await loadLog(
|
|
changelistBuild["id"], "test_results/0/logs/results.json/0")));
|
|
}
|
|
}),
|
|
]);
|
|
|
|
// Check the build was properly downloaded.
|
|
final buildPath = "${tmpdir.path}/$build";
|
|
final buildDirectory = new Directory(buildPath);
|
|
if (!await buildDirectory.exists()) {
|
|
print("$bot: Build directory didn't exist");
|
|
return <Test>[];
|
|
}
|
|
|
|
// Load the run.json to find the named configuration.
|
|
final resultsFile = new File("$buildPath/results.json");
|
|
if (!await resultsFile.exists()) {
|
|
print("$bot: No results.json exists");
|
|
return <Test>[];
|
|
}
|
|
|
|
// Load the current results, the approved resutls, and the flakiness
|
|
// information.
|
|
final results = await loadResultsMapIfExists("$buildPath/results.json");
|
|
final flaky = await loadResultsMapIfExists("$buildPath/flaky.json");
|
|
final approvedResults =
|
|
await loadResultsMapIfExists("${tmpdir.path}/approved_results.json");
|
|
|
|
// TODO: Remove 2019-04-08: Discard any invalid pre-approvals made with a
|
|
// version of approve_results between 065910f0 and a13ac1b4. Pre-approving
|
|
// a new test could add pre-approvals with null configuration and null name.
|
|
approvedResults.remove("null:null");
|
|
|
|
// Construct an object for every test containing its current result,
|
|
// what the last approved result was, and whether it's flaky.
|
|
final tests = <Test>[];
|
|
final testResults = changeId != null ? tryResults : results;
|
|
for (final key in testResults.keys) {
|
|
final baselineResult = changeId != null ? results[key] : null;
|
|
final testResult = testResults[key];
|
|
final approvedResult = approvedResults[key];
|
|
final flakiness = flaky[key];
|
|
final test =
|
|
new Test(bot, baselineResult, testResult, approvedResult, flakiness);
|
|
tests.add(test);
|
|
}
|
|
// Add in approvals whose test was no longer in the results.
|
|
for (final key in approvedResults.keys) {
|
|
if (testResults.containsKey(key)) continue;
|
|
final baselineResult = changeId != null ? results[key] : null;
|
|
final approvedResult = approvedResults[key];
|
|
final flakiness = flaky[key];
|
|
final test =
|
|
new Test(bot, baselineResult, null, approvedResult, flakiness);
|
|
tests.add(test);
|
|
}
|
|
if (options["verbose"]) {
|
|
print("Loaded $bot (${tests.length} tests).");
|
|
}
|
|
return tests;
|
|
} finally {
|
|
// Always clean up the temporary directory when we don't need it.
|
|
await tmpdir.delete(recursive: true);
|
|
}
|
|
}
|
|
|
|
Future<Map<String, dynamic>> loadJsonPrefixedAPI(String url) async {
|
|
final client = new HttpClient();
|
|
try {
|
|
final request = await client
|
|
.getUrl(Uri.parse(url))
|
|
.timeout(const Duration(seconds: 30));
|
|
final response = await request.close().timeout(const Duration(seconds: 30));
|
|
if (response.statusCode != HttpStatus.ok) {
|
|
throw new Exception("Failed to request $url: ${response.statusCode}");
|
|
}
|
|
final text = await response
|
|
.cast<List<int>>()
|
|
.transform(utf8.decoder)
|
|
.join()
|
|
.timeout(const Duration(seconds: 30));
|
|
return jsonDecode(text.substring(5 /* ")]}'\n" */));
|
|
} finally {
|
|
client.close();
|
|
}
|
|
}
|
|
|
|
Future<Map<String, dynamic>> loadChangelistDetails(
|
|
String gerritHost, String changeId) async {
|
|
// ?O=516714 requests the revisions field.
|
|
final url = "https://$gerritHost/changes/$changeId/detail?O=516714";
|
|
return await loadJsonPrefixedAPI(url);
|
|
}
|
|
|
|
main(List<String> args) async {
|
|
final parser = new ArgParser();
|
|
parser.addFlag("automated-approver",
|
|
help: "Record the approval as done by an automated process.",
|
|
negatable: false);
|
|
parser.addMultiOption("bot",
|
|
abbr: "b",
|
|
help: "Select the bots matching the glob pattern [option is repeatable]",
|
|
splitCommas: false);
|
|
parser.addFlag("help", help: "Show the program usage.", negatable: false);
|
|
parser.addFlag("failures-only",
|
|
help: "Approve failures only.", negatable: false);
|
|
parser.addFlag("list",
|
|
abbr: "l", help: "List the available bots.", negatable: false);
|
|
parser.addFlag("no",
|
|
abbr: "n",
|
|
help: "Show changed results but don't approve.",
|
|
negatable: false);
|
|
parser.addOption("preapprove",
|
|
abbr: "p", help: "Preapprove the new failures in a gerrit CL.");
|
|
parser.addFlag("successes-only",
|
|
help: "Approve successes only.", negatable: false);
|
|
parser.addFlag("verbose",
|
|
abbr: "v", help: "Describe asynchronous operations.", negatable: false);
|
|
parser.addFlag("yes",
|
|
abbr: "y", help: "Approve the results.", negatable: false);
|
|
parser.addOption("table",
|
|
abbr: "T",
|
|
help: "Select table format.",
|
|
allowed: ["markdown", "indent"],
|
|
defaultsTo: "markdown");
|
|
|
|
final options = parser.parse(args);
|
|
if ((options["preapprove"] == null &&
|
|
options["bot"].isEmpty &&
|
|
!options["list"]) ||
|
|
options["help"]) {
|
|
print("""
|
|
Usage: approve_results.dart [OPTION]...
|
|
List tests whose results are different from the previously approved results, and
|
|
ask whether to update the currently approved results, turning the bots green.
|
|
|
|
See the documentation at https://goto.google.com/dart-status-file-free-workflow
|
|
|
|
The options are as follows:
|
|
|
|
${parser.usage}""");
|
|
return;
|
|
}
|
|
|
|
if (options["no"] && options["yes"]) {
|
|
stderr.writeln("The --no and --yes options are mutually incompatible");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
if (options.rest.isNotEmpty) {
|
|
stderr.writeln("Unexpected extra argument: ${options.rest.first}");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
// Locate gsutil.py.
|
|
gsutilPy =
|
|
Platform.script.resolve("../third_party/gsutil/gsutil.py").toFilePath();
|
|
|
|
// Load the list of bots according to the test matrix.
|
|
final testMatrixPath =
|
|
Platform.script.resolve("bots/test_matrix.json").toFilePath();
|
|
final testMatrix = jsonDecode(await new File(testMatrixPath).readAsString());
|
|
final builderConfigurations = testMatrix["builder_configurations"];
|
|
final testMatrixBots = <String>[];
|
|
for (final builderConfiguration in builderConfigurations) {
|
|
final steps = builderConfiguration["steps"];
|
|
// Only consider bots that use tools/test.py or custom test runners.
|
|
if (!steps.any((step) =>
|
|
step["script"] == null ||
|
|
step["script"] == "tools/test.py" ||
|
|
step["testRunner"] == true)) {
|
|
continue;
|
|
}
|
|
final builders = builderConfiguration["builders"].cast<String>();
|
|
testMatrixBots.addAll(builders);
|
|
}
|
|
|
|
// Load the list of bots that have data in cloud storage.
|
|
if (options["verbose"]) {
|
|
print("Loading list of bots...");
|
|
}
|
|
final botsWithData = (await listBots())
|
|
.where((bot) => !bot.endsWith("-try"))
|
|
.where((bot) => !bot.endsWith("-dev"))
|
|
.where((bot) => !bot.endsWith("-stable"));
|
|
if (options["verbose"]) {
|
|
print("Loaded list of bots.");
|
|
}
|
|
|
|
// The currently active bots are the bots both mentioned in the test matrix
|
|
// and that have results in cloud storage.
|
|
final allBots = new Set<String>.from(testMatrixBots)
|
|
.intersection(new Set<String>.from(botsWithData))
|
|
.toList()
|
|
..sort();
|
|
|
|
// List the currently active bots if requested.
|
|
if (options["list"]) {
|
|
for (final bot in allBots) {
|
|
print(bot);
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Determine which builders have run for the changelist.
|
|
final changelistBuilds = <String, Map<String, dynamic>>{};
|
|
final isPreapproval = options["preapprove"] != null;
|
|
String changeId;
|
|
if (isPreapproval) {
|
|
if (options["verbose"]) {
|
|
print("Loading changelist details...");
|
|
}
|
|
final gerritHost = "dart-review.googlesource.com";
|
|
final gerritProject = "sdk";
|
|
final prefix = "https://$gerritHost/c/$gerritProject/+/";
|
|
final gerrit = options["preapprove"];
|
|
if (!gerrit.startsWith(prefix)) {
|
|
stderr.writeln("error: $gerrit doesn't start with $prefix");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
final components = gerrit.substring(prefix.length).split("/");
|
|
if (!((components.length == 1 && int.tryParse(components[0]) != null) ||
|
|
(components.length == 2 &&
|
|
int.tryParse(components[0]) != null &&
|
|
int.tryParse(components[1]) != null))) {
|
|
stderr.writeln("error: $gerrit must be in the form of "
|
|
"$prefix<changelist> or $prefix<changelist>/<patchset>");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
final changelist = int.parse(components[0]);
|
|
final details =
|
|
await loadChangelistDetails(gerritHost, changelist.toString());
|
|
changeId = details["change_id"];
|
|
final patchset = 2 <= components.length
|
|
? int.parse(components[1])
|
|
: details["revisions"][details["current_revision"]]["_number"];
|
|
if (2 <= components.length) {
|
|
print("Using Change-Id $changeId patchset $patchset");
|
|
} else {
|
|
print("Using Change-Id $changeId with the latest patchset $patchset");
|
|
}
|
|
if (options["verbose"]) {
|
|
print("Loading list of try runs...");
|
|
}
|
|
final buildset = "buildset:patch/gerrit/$gerritHost/$changelist/$patchset";
|
|
|
|
Future<Map<String, dynamic>> searchBuilds(String cursor) async {
|
|
final url = Uri.parse(
|
|
"https://cr-buildbucket.appspot.com/_ah/api/buildbucket/v1/search"
|
|
"?bucket=luci.dart.try"
|
|
"&tag=${Uri.encodeComponent(buildset)}"
|
|
"&fields=builds(id%2Ctags%2Cstatus%2Cstarted_ts),next_cursor"
|
|
"&start_cursor=$cursor");
|
|
final client = new HttpClient();
|
|
final request =
|
|
await client.getUrl(url).timeout(const Duration(seconds: 30));
|
|
final response =
|
|
await request.close().timeout(const Duration(seconds: 30));
|
|
if (response.statusCode != HttpStatus.ok) {
|
|
throw new Exception("Failed to request try runs for $gerrit");
|
|
}
|
|
final Map<String, dynamic> object = await response
|
|
.cast<List<int>>()
|
|
.transform(new Utf8Decoder())
|
|
.transform(new JsonDecoder())
|
|
.first
|
|
.timeout(const Duration(seconds: 30));
|
|
client.close();
|
|
return object;
|
|
}
|
|
|
|
var cursor = "";
|
|
final builds = [];
|
|
do {
|
|
final object = await searchBuilds(cursor);
|
|
if (cursor.isEmpty && object["builds"] == null) {
|
|
stderr.writeln(
|
|
"error: $prefix$changelist has no try runs for patchset $patchset");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
builds.addAll(object["builds"]);
|
|
cursor = object["next_cursor"];
|
|
} while (cursor != null);
|
|
|
|
// Prefer the newest completed build.
|
|
Map<String, dynamic> preferredBuild(
|
|
Map<String, dynamic> a, Map<String, dynamic> b) {
|
|
if (a != null && b == null) return a;
|
|
if (a == null && b != null) return b;
|
|
if (a != null && b != null) {
|
|
if (a["status"] == "COMPLETED" && b["status"] != "COMPLETED") return a;
|
|
if (a["status"] != "COMPLETED" && b["status"] == "COMPLETED") return b;
|
|
if (a["started_ts"] == null && b["started_ts"] != null) return a;
|
|
if (a["started_ts"] != null && b["started_ts"] == null) return b;
|
|
if (a["started_ts"] != null && b["started_ts"] != null) {
|
|
if (int.parse(a["started_ts"]) > int.parse(b["started_ts"])) return a;
|
|
if (int.parse(a["started_ts"]) < int.parse(b["started_ts"])) return b;
|
|
}
|
|
}
|
|
return b;
|
|
}
|
|
|
|
for (final build in builds) {
|
|
final tags = (build["tags"] as List<dynamic>).cast<String>();
|
|
final builder = tags
|
|
.firstWhere((tag) => tag.startsWith("builder:"))
|
|
.substring("builder:".length);
|
|
final ciBuilder = builder.replaceFirst(new RegExp("-try\$"), "");
|
|
if (!allBots.contains(ciBuilder)) {
|
|
continue;
|
|
}
|
|
changelistBuilds[ciBuilder] =
|
|
preferredBuild(changelistBuilds[ciBuilder], build);
|
|
}
|
|
if (options["verbose"]) {
|
|
print("Loaded list of try runs.");
|
|
}
|
|
}
|
|
final changelistBuilders = new Set<String>.from(changelistBuilds.keys);
|
|
|
|
// Select all the bots matching the glob patterns,
|
|
final finalBotList =
|
|
options["preapprove"] != null ? changelistBuilders : allBots;
|
|
final botPatterns = options["preapprove"] != null && options["bot"].isEmpty
|
|
? ["*"]
|
|
: options["bot"];
|
|
final bots = new Set<String>();
|
|
for (final botPattern in botPatterns) {
|
|
final glob = new Glob(botPattern);
|
|
bool any = false;
|
|
for (final bot in finalBotList) {
|
|
if (glob.matches(bot)) {
|
|
bots.add(bot);
|
|
any = true;
|
|
}
|
|
}
|
|
if (!any) {
|
|
stderr.writeln("error: No bots matched pattern: $botPattern");
|
|
stderr.writeln("Try --list to get the list of bots, or --help for help");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
}
|
|
for (final bot in bots) {
|
|
print("Selected bot: $bot");
|
|
}
|
|
|
|
// Error out if any of the requested try runs are incomplete.
|
|
bool anyIncomplete = false;
|
|
for (final bot in bots) {
|
|
if (options["preapprove"] != null &&
|
|
changelistBuilds[bot]["status"] != "COMPLETED") {
|
|
stderr.writeln("error: The try run for $bot isn't complete yet: " +
|
|
changelistBuilds[bot]["status"]);
|
|
anyIncomplete = true;
|
|
}
|
|
}
|
|
if (anyIncomplete) {
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
|
|
// Load all the latest results for the selected bots, as well as flakiness
|
|
// data, and the set of currently approved results. Each bot's latest build
|
|
// is downloaded in parallel to make this phase faster.
|
|
final testListFutures = <Future<List<Test>>>[];
|
|
final noResultsBuilds = new SplayTreeMap<String, String>();
|
|
for (final String bot in bots) {
|
|
testListFutures.add(new Future(() async {
|
|
try {
|
|
return await loadResultsFromBot(
|
|
bot, options, changeId, changelistBuilds[bot]);
|
|
} on NoResultsException catch (e) {
|
|
print(
|
|
"Error: Failed to find results for $bot build <${e.buildUrl}>: $e");
|
|
noResultsBuilds[bot] = e.buildUrl;
|
|
return <Test>[];
|
|
}
|
|
}));
|
|
}
|
|
|
|
// Collect all the tests from the synchronous downloads.
|
|
final tests = <Test>[];
|
|
for (final testList in await Future.wait(testListFutures)) {
|
|
tests.addAll(testList);
|
|
}
|
|
tests.sort();
|
|
print("");
|
|
|
|
// Compute statistics and the set of interesting tests.
|
|
final flakyTestsCount =
|
|
tests.where((test) => test.resultData != null && test.isFlake).length;
|
|
final failingTestsCount = tests
|
|
.where(
|
|
(test) => test.resultData != null && !test.isFlake && !test.matches)
|
|
.length;
|
|
final differentTests = tests
|
|
.where((test) =>
|
|
(isPreapproval ? test.isDifferent : !test.isApproved) &&
|
|
!test.isFlake)
|
|
.toList();
|
|
final selectedTests = differentTests
|
|
.where((test) => !(test.matches
|
|
? options["failures-only"]
|
|
: options["successes-only"]))
|
|
.toList();
|
|
final fixedTests = selectedTests.where((test) => test.matches).toList();
|
|
final brokenTests = selectedTests.where((test) => !test.matches).toList();
|
|
|
|
// Find out which bots have multiple configurations.
|
|
final configurationsForBots = <String, Set<String>>{};
|
|
for (final test in tests) {
|
|
var configurationSet = configurationsForBots[test.bot];
|
|
if (configurationSet == null) {
|
|
configurationsForBots[test.bot] = configurationSet = new Set<String>();
|
|
}
|
|
configurationSet.add(test.configuration);
|
|
}
|
|
|
|
// Compute a nice displayed name for the bot and configuration. If the bot
|
|
// only has a single configuration, then only mention the bot. Otherwise,
|
|
// remove the redundant parts from configuration and present it compactly.
|
|
// This is needed to avoid the tables becoming way too large.
|
|
String getBotDisplayName(String bot, String configuration) {
|
|
if (configurationsForBots[bot].length == 1) {
|
|
return bot;
|
|
} else {
|
|
final simpleConfig = simplifyNamedConfiguration(bot, configuration);
|
|
return "$bot/$simpleConfig";
|
|
}
|
|
}
|
|
|
|
// Compute the width of the fields in the below tables.
|
|
final unapprovedBots = new Set<String>();
|
|
int longestBot = "BOT/CONFIG".length;
|
|
int longestTest = "TEST".length;
|
|
int longestResult = "RESULT".length;
|
|
int longestExpected = "EXPECTED".length;
|
|
for (final test in selectedTests) {
|
|
unapprovedBots.add(test.bot);
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
longestBot = max(longestBot, botDisplayName.length);
|
|
longestTest = max(longestTest, test.name.length);
|
|
longestResult = max(longestResult, test.result.length);
|
|
longestExpected = max(longestExpected, test.expected.length);
|
|
}
|
|
longestTest = min(longestTest, 120); // Some tests names are extremely long.
|
|
|
|
// Table of lists that now succeed.
|
|
if (fixedTests.isNotEmpty) {
|
|
print("The following tests are now succeeding:\n");
|
|
if (options["table"] == "markdown") {
|
|
print("| ${'BOT/CONFIG'.padRight(longestBot)} "
|
|
"| ${'TEST'.padRight(longestTest)} |");
|
|
print("| ${'-' * longestBot} "
|
|
"| ${'-' * longestTest} |");
|
|
} else if (options["table"] == "indent") {
|
|
print("${'BOT/CONFIG'.padRight(longestBot)} "
|
|
"TEST");
|
|
}
|
|
for (final test in fixedTests) {
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
if (options["table"] == "markdown") {
|
|
print("| ${botDisplayName.padRight(longestBot)} "
|
|
"| ${test.name.padRight(longestTest)} |");
|
|
} else if (options["table"] == "indent") {
|
|
print("${botDisplayName.padRight(longestBot)} "
|
|
"${test.name}");
|
|
}
|
|
}
|
|
print("");
|
|
}
|
|
|
|
/// Table of lists that now fail.
|
|
if (brokenTests.isNotEmpty) {
|
|
print("The following tests are now failing:\n");
|
|
if (options["table"] == "markdown") {
|
|
print("| ${'BOT'.padRight(longestBot)} "
|
|
"| ${'TEST'.padRight(longestTest)} "
|
|
"| ${'RESULT'.padRight(longestResult)} "
|
|
"| ${'EXPECTED'.padRight(longestExpected)} | ");
|
|
print("| ${'-' * longestBot} "
|
|
"| ${'-' * longestTest} "
|
|
"| ${'-' * longestResult} "
|
|
"| ${'-' * longestExpected} | ");
|
|
} else if (options["table"] == "indent") {
|
|
print("${'BOT'.padRight(longestBot)} "
|
|
"${'TEST'.padRight(longestTest)} "
|
|
"${'RESULT'.padRight(longestResult)} "
|
|
"EXPECTED");
|
|
}
|
|
for (final test in brokenTests) {
|
|
final botDisplayName = getBotDisplayName(test.bot, test.configuration);
|
|
if (options["table"] == "markdown") {
|
|
print("| ${botDisplayName.padRight(longestBot)} "
|
|
"| ${test.name.padRight(longestTest)} "
|
|
"| ${test.result.padRight(longestResult)} "
|
|
"| ${test.expected.padRight(longestExpected)} |");
|
|
} else if (options["table"] == "indent") {
|
|
print("${botDisplayName.padRight(longestBot)} "
|
|
"${test.name.padRight(longestTest)} "
|
|
"${test.result.padRight(longestResult)} "
|
|
"${test.expected}");
|
|
}
|
|
}
|
|
print("");
|
|
}
|
|
|
|
// Provide statistics on how well the bots are doing.
|
|
void statistic(int numerator, int denominator, String what) {
|
|
double percent = numerator / denominator * 100.0;
|
|
String percentString = percent.toStringAsFixed(2) + "%";
|
|
print("$numerator of $denominator $what ($percentString)");
|
|
}
|
|
|
|
statistic(failingTestsCount, tests.length, "tests are failing");
|
|
statistic(flakyTestsCount, tests.length, "tests are flaky");
|
|
statistic(
|
|
fixedTests.length, tests.length, "tests were fixed since last approval");
|
|
statistic(brokenTests.length, tests.length,
|
|
"tests were broken since last approval");
|
|
|
|
// Warn about any builders where results weren't available.
|
|
if (noResultsBuilds.isNotEmpty) {
|
|
print("");
|
|
noResultsBuilds.forEach((String builder, String buildUrl) {
|
|
print("Warning: No results were found for $builder: <$buildUrl>");
|
|
});
|
|
print("Warning: Builders without results are usually due to infrastructure "
|
|
"issues, please have a closer look at the affected builders and try "
|
|
"the build again.");
|
|
}
|
|
|
|
// Stop if there's nothing to do.
|
|
if (unapprovedBots.isEmpty) {
|
|
print("\nEvery test result has already been approved.");
|
|
return;
|
|
}
|
|
|
|
// Stop if this is a dry run.
|
|
if (options["no"]) {
|
|
if (selectedTests.length == 1) {
|
|
print("1 test has a changed result and needs approval");
|
|
} else {
|
|
print("${selectedTests.length} "
|
|
"tests have changed results and need approval");
|
|
}
|
|
return;
|
|
}
|
|
|
|
// Confirm the approval if run interactively.
|
|
if (!options["yes"]) {
|
|
print("");
|
|
print("Note: It is assumed bugs have been filed about the above failures "
|
|
"before they are approved here.");
|
|
if (brokenTests.isNotEmpty) {
|
|
final builderPlural = bots.length == 1 ? "builder" : "builders";
|
|
final tryBuilders = isPreapproval ? "try$builderPlural" : builderPlural;
|
|
final tryCommit = isPreapproval ? "tryrun" : "commit";
|
|
print("Note: Approving the failures will turn the "
|
|
"$tryBuilders green on the next $tryCommit.");
|
|
}
|
|
while (true) {
|
|
final approve = isPreapproval ? "pre-approve" : "approve";
|
|
stdout.write("Do you want to $approve? (yes/no) [yes] ");
|
|
final line = stdin.readLineSync();
|
|
// End of file condition is considered no.
|
|
if (line == null) {
|
|
print("n");
|
|
return;
|
|
}
|
|
if (line.toLowerCase() == "n" || line.toLowerCase() == "no") {
|
|
return;
|
|
}
|
|
if (line == "" ||
|
|
line.toLowerCase() == "y" ||
|
|
line.toLowerCase() == "yes") {
|
|
break;
|
|
}
|
|
}
|
|
} else {
|
|
print("Note: It is assumed bugs have been filed about the above failures.");
|
|
}
|
|
print("");
|
|
|
|
// Log who approved these results.
|
|
final username =
|
|
(options["automated-approver"] ? "automatic-approval" : null) ??
|
|
Platform.environment["LOGNAME"] ??
|
|
Platform.environment["USER"] ??
|
|
Platform.environment["USERNAME"];
|
|
if (username == null || username == "") {
|
|
stderr.writeln("error: Your identity could not be established. "
|
|
"Please set one of the LOGNAME, USER, USERNAME environment variables.");
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
final nowDate = new DateTime.now().toUtc();
|
|
final now = nowDate.toIso8601String();
|
|
|
|
// Deep clones a decoded json object.
|
|
dynamic deepClone(dynamic object) {
|
|
if (object is Map<String, dynamic>) {
|
|
final result = <String, dynamic>{};
|
|
for (final key in object.keys) {
|
|
result[key] = deepClone(object[key]);
|
|
}
|
|
return result;
|
|
} else if (object is List<dynamic>) {
|
|
final result = <dynamic>[];
|
|
for (final value in object) {
|
|
result.add(deepClone(value));
|
|
}
|
|
return result;
|
|
} else {
|
|
return object;
|
|
}
|
|
}
|
|
|
|
// Build the new approval data with the changes in test results applied.
|
|
final newApprovalsForBuilders = <String, Map<String, Map<String, dynamic>>>{};
|
|
|
|
if (isPreapproval) {
|
|
// Import all the existing approval data, keeping tests that don't exist
|
|
// anymore.
|
|
for (final test in tests) {
|
|
if (test.approvedResultData == null) continue;
|
|
final approvalData = deepClone(test.approvedResultData);
|
|
// TODO(https://github.com/dart-lang/sdk/issues/36279): Remove needless
|
|
// fields that shouldn't be in the approvals data. Remove this 2019-04-03.
|
|
approvalData.remove("bot_name");
|
|
approvalData.remove("builder_name");
|
|
approvalData.remove("build_number");
|
|
approvalData.remove("changed");
|
|
approvalData.remove("commit_hash");
|
|
approvalData.remove("commit_time");
|
|
approvalData.remove("commit_hash");
|
|
approvalData.remove("flaky");
|
|
approvalData.remove("previous_build_number");
|
|
approvalData.remove("previous_commit_hash");
|
|
approvalData.remove("previous_commit_time");
|
|
approvalData.remove("previous_flaky");
|
|
approvalData.remove("previous_result");
|
|
approvalData.remove("time_ms");
|
|
// Discard all the existing pre-approvals for this changelist.
|
|
final preapprovals =
|
|
approvalData.putIfAbsent("preapprovals", () => <String, dynamic>{});
|
|
preapprovals.remove(changeId);
|
|
final newApprovals = newApprovalsForBuilders.putIfAbsent(
|
|
test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
|
|
newApprovals[test.key] = approvalData;
|
|
}
|
|
|
|
// Pre-approve all the regressions (no need to pre-approve fixed tests).
|
|
for (final test in brokenTests) {
|
|
final newApprovals = newApprovalsForBuilders.putIfAbsent(
|
|
test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
|
|
final approvalData =
|
|
newApprovals.putIfAbsent(test.key, () => <String, dynamic>{});
|
|
approvalData["name"] = test.name;
|
|
approvalData["configuration"] = test.configuration;
|
|
approvalData["suite"] = test.resultData["suite"];
|
|
approvalData["test_name"] = test.resultData["test_name"];
|
|
final preapprovals =
|
|
approvalData.putIfAbsent("preapprovals", () => <String, dynamic>{});
|
|
final preapproval =
|
|
preapprovals.putIfAbsent(changeId, () => <String, dynamic>{});
|
|
preapproval["from"] = test.approvedResult;
|
|
preapproval["result"] = test.result;
|
|
preapproval["matches"] = test.matches;
|
|
preapproval["expected"] = test.expected;
|
|
preapproval["preapprover"] = username;
|
|
preapproval["preapproved_at"] = now;
|
|
preapproval["expires"] =
|
|
nowDate.add(const Duration(days: 30)).toIso8601String();
|
|
}
|
|
} else {
|
|
// Import all the existing approval data for tests, removing tests that
|
|
// don't exist anymore unless they have pre-approvals.
|
|
for (final test in tests) {
|
|
if (test.approvedResultData == null) continue;
|
|
if (test.result == null &&
|
|
(test.approvedResultData["preapprovals"] ?? <dynamic>[]).isEmpty) {
|
|
continue;
|
|
}
|
|
final approvalData = deepClone(test.approvedResultData);
|
|
// TODO(https://github.com/dart-lang/sdk/issues/36279): Remove needless
|
|
// fields that shouldn't be in the approvals data. Remove this 2019-04-03.
|
|
approvalData.remove("bot_name");
|
|
approvalData.remove("builder_name");
|
|
approvalData.remove("build_number");
|
|
approvalData.remove("changed");
|
|
approvalData.remove("commit_hash");
|
|
approvalData.remove("commit_time");
|
|
approvalData.remove("commit_hash");
|
|
approvalData.remove("flaky");
|
|
approvalData.remove("previous_build_number");
|
|
approvalData.remove("previous_commit_hash");
|
|
approvalData.remove("previous_commit_time");
|
|
approvalData.remove("previous_flaky");
|
|
approvalData.remove("previous_result");
|
|
approvalData.remove("time_ms");
|
|
approvalData.putIfAbsent("preapprovals", () => <String, dynamic>{});
|
|
final newApprovals = newApprovalsForBuilders.putIfAbsent(
|
|
test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
|
|
newApprovals[test.key] = approvalData;
|
|
}
|
|
|
|
// Approve the changes in test results.
|
|
for (final test in selectedTests) {
|
|
final newApprovals = newApprovalsForBuilders.putIfAbsent(
|
|
test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
|
|
final approvalData =
|
|
newApprovals.putIfAbsent(test.key, () => <String, dynamic>{});
|
|
approvalData["name"] = test.name;
|
|
approvalData["configuration"] = test.configuration;
|
|
approvalData["suite"] = test.resultData["suite"];
|
|
approvalData["test_name"] = test.resultData["test_name"];
|
|
approvalData["result"] = test.result;
|
|
approvalData["expected"] = test.expected;
|
|
approvalData["matches"] = test.matches;
|
|
approvalData["approver"] = username;
|
|
approvalData["approved_at"] = now;
|
|
approvalData.putIfAbsent("preapprovals", () => <String, dynamic>{});
|
|
}
|
|
}
|
|
|
|
// Reconstruct the old approvals so we can double check there was no race
|
|
// condition when uploading.
|
|
final oldApprovalsForBuilders = <String, Map<String, Map<String, dynamic>>>{};
|
|
for (final test in tests) {
|
|
if (test.approvedResultData == null) continue;
|
|
final oldApprovals = oldApprovalsForBuilders.putIfAbsent(
|
|
test.bot, () => new SplayTreeMap<String, Map<String, dynamic>>());
|
|
oldApprovals[test.key] = test.approvedResultData;
|
|
}
|
|
for (final builder in newApprovalsForBuilders.keys) {
|
|
oldApprovalsForBuilders.putIfAbsent(
|
|
builder, () => <String, Map<String, dynamic>>{});
|
|
}
|
|
|
|
// Update approved_results.json for each builder with unapproved changes.
|
|
final outDirectory =
|
|
await Directory.systemTemp.createTemp("approved_results.");
|
|
bool raceCondition = false;
|
|
try {
|
|
print("Uploading approved results...");
|
|
final futures = <Future>[];
|
|
for (final String builder in newApprovalsForBuilders.keys) {
|
|
final approvals = newApprovalsForBuilders[builder].values;
|
|
final localPath = "${outDirectory.path}/$builder.json";
|
|
await new File(localPath).writeAsString(
|
|
approvals.map((approval) => jsonEncode(approval) + "\n").join(""));
|
|
final remotePath =
|
|
"$approvedResultsStoragePath/$builder/approved_results.json";
|
|
futures.add(new Future(() async {
|
|
if (!options["yes"]) {
|
|
if (options["verbose"]) {
|
|
print("Checking for race condition on $builder...");
|
|
}
|
|
final oldApprovedResults = oldApprovalsForBuilders[builder];
|
|
final oldApprovalPath = "${outDirectory.path}/$builder.json.old";
|
|
await cpGsutil(remotePath, oldApprovalPath);
|
|
final checkApprovedResults =
|
|
await loadResultsMapIfExists(oldApprovalPath);
|
|
if (!isIdenticalApprovals(oldApprovedResults, checkApprovedResults)) {
|
|
print("error: Race condition: "
|
|
"$builder approvals have changed, please try again.");
|
|
raceCondition = true;
|
|
return;
|
|
}
|
|
}
|
|
if (options["verbose"]) {
|
|
print("Uploading approved results for $builder...");
|
|
}
|
|
await cpGsutil(localPath, remotePath);
|
|
print("Uploaded approved results for $builder");
|
|
}));
|
|
}
|
|
await Future.wait(futures);
|
|
if (raceCondition) {
|
|
exitCode = 1;
|
|
print("error: Somebody else has approved, please try again");
|
|
return;
|
|
}
|
|
if (brokenTests.isNotEmpty) {
|
|
final approved = isPreapproval ? "pre-approved" : "approved";
|
|
final commit = isPreapproval ? "tryrun" : "commit";
|
|
print("Successfully $approved results, the next $commit "
|
|
"will turn builders green");
|
|
} else {
|
|
print("Successfully approved results");
|
|
}
|
|
} finally {
|
|
await outDirectory.delete(recursive: true);
|
|
}
|
|
}
|