mirror of
https://github.com/dart-lang/sdk
synced 2024-09-05 00:13:50 +00:00
![Josh Soref](/assets/img/avatar_default.png)
Closes https://github.com/dart-lang/sdk/pull/50880 TEST=ci GitOrigin-RevId: ea8ecf4b7b921de0549df325be1a1f7879e86b72 Change-Id: I39315609a8390b012196a08c9ce4a9a6a50b1558 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/278061 Reviewed-by: Slava Egorov <vegorov@google.com> Commit-Queue: Michael Thomsen <mit@google.com> Reviewed-by: Michael Thomsen <mit@google.com>
409 lines
11 KiB
Dart
409 lines
11 KiB
Dart
#!/usr/bin/env dart
|
|
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:args/args.dart';
|
|
import 'package:glob/glob.dart';
|
|
|
|
final parser = ArgParser()
|
|
..addMultiOption('bot',
|
|
abbr: 'b',
|
|
help: 'Select the bots matching the glob pattern [option is repeatable]',
|
|
splitCommas: false)
|
|
..addFlag('verbose', abbr: 'v', help: 'Verbose output.', negatable: false)
|
|
..addFlag('help', help: 'Show the program usage.', negatable: false);
|
|
|
|
void printUsage() {
|
|
print("""
|
|
Usage: ${Platform.executable} ${Platform.script} [OLDER_COMMIT] [NEWER_COMMIT]
|
|
|
|
The options are as follows:
|
|
|
|
${parser.usage}""");
|
|
}
|
|
|
|
late bool verbose;
|
|
|
|
main(List<String> args) async {
|
|
final options = parser.parse(args);
|
|
if (options["help"]) {
|
|
printUsage();
|
|
return;
|
|
}
|
|
|
|
final commits = options.rest;
|
|
if (commits.length < 2) {
|
|
print('Need to supply at least two commits.');
|
|
printUsage();
|
|
exitCode = 1;
|
|
return;
|
|
}
|
|
verbose = options['verbose'] ?? false;
|
|
|
|
final globs = List<Glob>.from(options["bot"].map((pattern) => Glob(pattern)));
|
|
final vmBuilders = loadVmBuildersFromTestMatrix(globs);
|
|
|
|
final futures = <Future<List<Result>>>[];
|
|
for (final commit in commits) {
|
|
final DateTime date = await getDateOfCommit(commit);
|
|
futures.add(getResults(commit, date, vmBuilders));
|
|
}
|
|
|
|
final results = await Future.wait(futures);
|
|
for (int i = 0; i < results.length - 1; i++) {
|
|
final commitB = commits[i];
|
|
final commitA = commits[i + 1];
|
|
|
|
print('\nResult changes between $commitB -> $commitA:');
|
|
final commonGroups =
|
|
buildCommonGroups(commitA, commitB, results[i], results[i + 1]);
|
|
for (final commonGroup in commonGroups) {
|
|
final builders = commonGroup.builders;
|
|
|
|
print('');
|
|
for (final group in commonGroup.groups) {
|
|
final diff = group.diffs.first;
|
|
print('${group.test} ${diff.before} -> ${diff.after}');
|
|
}
|
|
for (final b in extractBuilderPattern(builders)) {
|
|
print(' on $b');
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Future<DateTime> getDateOfCommit(String commit) async {
|
|
final result = await Process.run(
|
|
'git', ['show', '-s', '--format=%cd', '--date=iso-strict', commit]);
|
|
if (result.exitCode != 0) {
|
|
print('Could not determine date of commit $commit. Git reported:\n');
|
|
print(result.stdout);
|
|
print(result.stderr);
|
|
exit(1);
|
|
}
|
|
return DateTime.parse(result.stdout.trim());
|
|
}
|
|
|
|
Future<List<Result>> getResults(
|
|
String commit, DateTime dateC, Set<String> builders) async {
|
|
final DateTime date0 = dateC.add(const Duration(hours: 24));
|
|
final DateTime date2 = dateC.subtract(const Duration(hours: 24));
|
|
final query = '''
|
|
SELECT commit_time, builder_name, build_number, name, result, expected FROM `dart-ci.results.results`
|
|
WHERE commit_hash="$commit"
|
|
AND matches=false
|
|
AND (_PARTITIONDATE = "${formatDate(date0)}" OR
|
|
_PARTITIONDATE = "${formatDate(dateC)}" OR
|
|
_PARTITIONDATE = "${formatDate(date2)}" )
|
|
AND (STARTS_WITH(builder_name, "vm-") OR
|
|
STARTS_WITH(builder_name, "app-") OR
|
|
STARTS_WITH(builder_name, "cross-"))
|
|
AND ((flaky is NULL) OR flaky=false)
|
|
ORDER BY name''';
|
|
|
|
final arguments = <String>[
|
|
'query',
|
|
'--format=prettyjson',
|
|
'--project_id=dart-ci',
|
|
'--nouse_legacy_sql',
|
|
'-n',
|
|
'1000000',
|
|
query,
|
|
];
|
|
if (verbose) {
|
|
print('Executing query:\n bq ${arguments.join(' ')}');
|
|
}
|
|
|
|
final result = await Process.run('bq', arguments);
|
|
if (result.exitCode == 0) {
|
|
File('$commit.json').writeAsStringSync(result.stdout);
|
|
final resultsForCommit = json.decode(result.stdout);
|
|
|
|
final results = <Result>[];
|
|
for (final Map<String, dynamic> result in resultsForCommit) {
|
|
final builderName = result['builder_name'];
|
|
if (!builders.contains(builderName)) {
|
|
continue;
|
|
}
|
|
|
|
final failure = Result(commit, builderName, result['build_number'],
|
|
result['name'], result['expected'], result['result']);
|
|
results.add(failure);
|
|
}
|
|
|
|
results.sort((Result a, Result b) {
|
|
final c = a.name.compareTo(b.name);
|
|
if (c != 0) return c;
|
|
return a.builderName.compareTo(b.builderName);
|
|
});
|
|
|
|
return results;
|
|
} else {
|
|
print('Running the following query failed:\nbq ${arguments.join(' ')}');
|
|
print('Exit code: ${result.exitCode}');
|
|
final stdout = result.stdout.trim();
|
|
if (stdout.length > 0) {
|
|
print('Stdout:\n$stdout');
|
|
}
|
|
final stderr = result.stderr.trim();
|
|
if (stderr.length > 0) {
|
|
print('Stderr:\n$stderr');
|
|
}
|
|
return <Result>[];
|
|
}
|
|
}
|
|
|
|
List<CommonGroup> buildCommonGroups(String commitA, String commitB,
|
|
List<Result> commitResults, List<Result> commitResultsBefore) {
|
|
// If a test has same outcome across many vm builders
|
|
final diffs = <Diff>[];
|
|
int i = 0;
|
|
int j = 0;
|
|
while (i < commitResultsBefore.length && j < commitResults.length) {
|
|
final a = commitResultsBefore[i];
|
|
final b = commitResults[j];
|
|
|
|
// Is a smaller than b, then we had a failure before and no longer one.
|
|
if (a.name.compareTo(b.name) < 0 ||
|
|
(a.name.compareTo(b.name) == 0 &&
|
|
a.builderName.compareTo(b.builderName) < 0)) {
|
|
diffs.add(Diff(a, null));
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
// Is b smaller than a, then we had no failure before but have one now.
|
|
if (b.name.compareTo(a.name) < 0 ||
|
|
(b.name.compareTo(a.name) == 0 &&
|
|
b.builderName.compareTo(a.builderName) < 0)) {
|
|
diffs.add(Diff(null, b));
|
|
j++;
|
|
continue;
|
|
}
|
|
|
|
// Else we must have the same name and builder.
|
|
if (a.name != b.name || a.builderName != b.builderName) throw 'BUG';
|
|
|
|
if (a.expected != b.expected || a.result != b.result) {
|
|
diffs.add(Diff(a, b));
|
|
}
|
|
i++;
|
|
j++;
|
|
}
|
|
|
|
while (i < commitResultsBefore.length) {
|
|
final a = commitResultsBefore[i++];
|
|
diffs.add(Diff(a, null));
|
|
}
|
|
|
|
while (j < commitResults.length) {
|
|
final b = commitResults[j++];
|
|
diffs.add(Diff(null, b));
|
|
}
|
|
|
|
// If a test has same outcome across many vm builders
|
|
final groups = <GroupedDiff>[];
|
|
int h = 0;
|
|
while (h < diffs.length) {
|
|
final d = diffs[h++];
|
|
final builders = <String>{}..add(d.builder);
|
|
final groupDiffs = <Diff>[d];
|
|
|
|
while (h < diffs.length) {
|
|
final nd = diffs[h];
|
|
if (d.test == nd.test) {
|
|
if (d.sameExpectationDifferenceAs(nd)) {
|
|
builders.add(nd.builder);
|
|
groupDiffs.add(nd);
|
|
h++;
|
|
continue;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
|
|
groups.add(GroupedDiff(d.test, builders.toList()..sort(), groupDiffs));
|
|
}
|
|
|
|
final commonGroups = <String, List<GroupedDiff>>{};
|
|
for (final group in groups) {
|
|
final key = group.builders.join(' ');
|
|
commonGroups.putIfAbsent(key, () => <GroupedDiff>[]).add(group);
|
|
}
|
|
|
|
final commonGroupList = commonGroups.values
|
|
.map((list) => CommonGroup(list.first.builders, list))
|
|
.toList();
|
|
commonGroupList
|
|
.sort((a, b) => a.builders.length.compareTo(b.builders.length));
|
|
return commonGroupList;
|
|
}
|
|
|
|
class CommonGroup {
|
|
final List<String> builders;
|
|
final List<GroupedDiff> groups;
|
|
CommonGroup(this.builders, this.groups);
|
|
}
|
|
|
|
class GroupedDiff {
|
|
final String test;
|
|
final List<String> builders;
|
|
final List<Diff> diffs;
|
|
|
|
GroupedDiff(this.test, this.builders, this.diffs);
|
|
}
|
|
|
|
class Diff {
|
|
final Result? before;
|
|
final Result? after;
|
|
|
|
Diff(this.before, this.after);
|
|
|
|
String get test => (before?.name ?? after?.name)!;
|
|
String get builder => (before?.builderName ?? after?.builderName)!;
|
|
|
|
bool sameExpectationDifferenceAs(Diff other) {
|
|
if ((before == null) != (other.before == null)) return false;
|
|
if ((after == null) != (other.after == null)) return false;
|
|
|
|
if (before != null) {
|
|
if (!before!.sameResult(other.before!)) return false;
|
|
}
|
|
if (after != null) {
|
|
if (!after!.sameResult(other.after!)) return false;
|
|
}
|
|
return true;
|
|
}
|
|
}
|
|
|
|
class Result {
|
|
final String commit;
|
|
final String builderName;
|
|
final String buildNumber;
|
|
final String name;
|
|
final String expected;
|
|
final String result;
|
|
|
|
Result(this.commit, this.builderName, this.buildNumber, this.name,
|
|
this.expected, this.result);
|
|
|
|
@override
|
|
String toString() => '(expected: $expected, actual: $result)';
|
|
|
|
bool sameResult(Result other) {
|
|
return name == other.name &&
|
|
expected == other.expected &&
|
|
result == other.result;
|
|
}
|
|
|
|
bool equals(other) {
|
|
if (other is Result) {
|
|
if (name != other.name) return false;
|
|
if (builderName != other.builderName) return false;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
@override
|
|
int get hashCode => name.hashCode ^ builderName.hashCode;
|
|
|
|
@override
|
|
bool operator ==(Object other) {
|
|
// TODO: implement ==
|
|
return super == other;
|
|
}
|
|
}
|
|
|
|
String currentDate() {
|
|
final timestamp = DateTime.now().toUtc().toIso8601String();
|
|
return timestamp.substring(0, timestamp.indexOf('T'));
|
|
}
|
|
|
|
Set<String> loadVmBuildersFromTestMatrix(List<Glob> globs) {
|
|
final contents = File('tools/bots/test_matrix.json').readAsStringSync();
|
|
final testMatrix = json.decode(contents);
|
|
|
|
final vmBuilders = <String>{};
|
|
for (final config in testMatrix['builder_configurations']) {
|
|
for (final builder in config['builders']) {
|
|
if (builder.startsWith('vm-') || builder.startsWith('app-')) {
|
|
vmBuilders.add(builder);
|
|
}
|
|
}
|
|
}
|
|
|
|
// This one is in the test_matrix.json but we don't run it on CI.
|
|
vmBuilders.remove('vm-kernel-asan-linux-release-ia32');
|
|
|
|
if (globs.isNotEmpty) {
|
|
vmBuilders.removeWhere((String builder) {
|
|
return !globs.any((Glob glob) => glob.matches(builder));
|
|
});
|
|
}
|
|
|
|
return vmBuilders;
|
|
}
|
|
|
|
List<String> extractBuilderPattern(List<String> builders) {
|
|
final all = Set<String>.from(builders);
|
|
|
|
String reduce(String builder, List<String> possibilities) {
|
|
for (final pos in possibilities) {
|
|
if (builder.contains(pos)) {
|
|
final existing = <String>[];
|
|
final available = <String>[];
|
|
for (final pos2 in possibilities) {
|
|
final builder2 = builder.replaceFirst(pos, pos2);
|
|
if (all.contains(builder2)) {
|
|
existing.add(builder2);
|
|
available.add(pos2);
|
|
}
|
|
}
|
|
if (existing.length > 1) {
|
|
all.removeAll(existing);
|
|
final replacement =
|
|
builder.replaceFirst(pos, '{${available.join(',')}}');
|
|
all.add(replacement);
|
|
return replacement;
|
|
}
|
|
}
|
|
}
|
|
return builder;
|
|
}
|
|
|
|
for (String builder in builders) {
|
|
if (all.contains(builder)) {
|
|
builder = reduce(builder, const ['debug', 'release', 'product']);
|
|
}
|
|
}
|
|
for (String builder in all.toList()) {
|
|
if (all.contains(builder)) {
|
|
builder = reduce(builder, const ['mac', 'linux', 'win']);
|
|
}
|
|
}
|
|
|
|
for (String builder in all.toList()) {
|
|
if (all.contains(builder)) {
|
|
builder = reduce(builder, const [
|
|
'ia32',
|
|
'x64',
|
|
'simarm',
|
|
'simarm64',
|
|
'arm',
|
|
'arm64',
|
|
]);
|
|
}
|
|
}
|
|
return all.toList()..sort();
|
|
}
|
|
|
|
String formatDate(DateTime date) {
|
|
final s = date.toIso8601String();
|
|
return s.substring(0, s.indexOf('T'));
|
|
}
|