languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-16 00:09:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

[infra] Support multiple named configurations in test infrastructure

Browse source 
This change allows to run test.dart and pkg/test_runner with multiple arguments
for the -n option to run tests for multiple configurations with one invocation.

Change-Id: If62e0bfc364460fa415c7f700f7e449b0de56987
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/122395
Commit-Queue: Karl Klose <karlklose@google.com>
Reviewed-by: William Hesse <whesse@google.com>
This commit is contained in:
Karl Klose 2019-11-08 08:06:36 +00:00 committed by commit-bot@chromium.org
parent b334ea8320
commit ec0d042955
4 changed files with 267 additions and 230 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

198
pkg/test_runner/lib/src/options.dart
View file

@ -679,10 +679,87 @@ compiler.''',
var progress = Progress.find(data["progress"] as String);
var nnbdMode = NnbdMode.find(data["nnbd"] as String);
void addConfiguration(Configuration innerConfiguration,
[String namedConfiguration]) {
var configuration = TestConfiguration(
configuration: innerConfiguration,
progress: progress,
selectors: selectors,
testList: data["test_list_contents"] as List<String>,
repeat: data["repeat"] as int,
batch: !(data["noBatch"] as bool),
batchDart2JS: data["dart2js_batch"] as bool,
copyCoreDumps: data["copy_coredumps"] as bool,
isVerbose: data["verbose"] as bool,
listTests: data["list"] as bool,
listStatusFiles: data["list_status_files"] as bool,
cleanExit: data["clean_exit"] as bool,
silentFailures: data["silent_failures"] as bool,
printTiming: data["time"] as bool,
printReport: data["report"] as bool,
reportInJson: data["report_in_json"] as bool,
resetBrowser: data["reset_browser_configuration"] as bool,
skipCompilation: data["skip_compilation"] as bool,
writeDebugLog: data["write_debug_log"] as bool,
writeResults: data["write_results"] as bool,
writeLogs: data["write_logs"] as bool,
drtPath: data["drt"] as String,
chromePath: data["chrome"] as String,
safariPath: data["safari"] as String,
firefoxPath: data["firefox"] as String,
dartPath: data["dart"] as String,
dartPrecompiledPath: data["dart_precompiled"] as String,
genSnapshotPath: data["gen-snapshot"] as String,
keepGeneratedFiles: data["keep_generated_files"] as bool,
taskCount: data["tasks"] as int,
shardCount: data["shards"] as int,
shard: data["shard"] as int,
stepName: data["step_name"] as String,
testServerPort: data["test_server_port"] as int,
testServerCrossOriginPort:
data['test_server_cross_origin_port'] as int,
testDriverErrorPort: data["test_driver_error_port"] as int,
localIP: data["local_ip"] as String,
sharedOptions: sharedOptions,
packages: data["packages"] as String,
packageRoot: data["package_root"] as String,
suiteDirectory: data["suite_dir"] as String,
outputDirectory: data["output_directory"] as String,
reproducingArguments:
_reproducingCommand(data, namedConfiguration != null),
fastTestsOnly: data["fast_tests"] as bool,
printPassingStdout: data["print_passing_stdout"] as bool);
if (configuration.validate()) {
result.add(configuration);
}
}
String namedConfigurationOption = data["named_configuration"] as String;
if (namedConfigurationOption != null) {
List<String> namedConfigurations = namedConfigurationOption.split(',');
var testMatrixFile = "tools/bots/test_matrix.json";
var testMatrix = TestMatrix.fromPath(testMatrixFile);
for (String namedConfiguration in namedConfigurations) {
var configuration = testMatrix.configurations.singleWhere(
(c) => c.name == namedConfiguration,
orElse: () => null);
if (configuration == null) {
var names = testMatrix.configurations
.map((configuration) => configuration.name)
.toList();
names.sort();
_fail('The named configuration "$namedConfiguration" does not exist.'
' The following configurations are available:\n'
' * ${names.join('\n * ')}');
}
addConfiguration(configuration);
}
return result;
}
// Expand runtimes.
for (var runtime in runtimes) {
// Start installing the runtime if needed.
// Expand architectures.
var architectures = data["arch"] as String;
if (architectures == "all") {
@ -700,82 +777,28 @@ compiler.''',
for (var modeName in modes.split(",")) {
var mode = Mode.find(modeName);
var system = System.find(data["system"] as String);
var namedConfiguration =
_namedConfiguration(data["named_configuration"] as String);
var innerConfiguration = namedConfiguration ??
Configuration("custom configuration", architecture, compiler,
mode, runtime, system,
nnbdMode: nnbdMode,
timeout: data["timeout"] as int,
enableAsserts: data["enable_asserts"] as bool,
useAnalyzerCfe: data["use_cfe"] as bool,
useAnalyzerFastaParser:
data["analyzer_use_fasta_parser"] as bool,
useBlobs: data["use_blobs"] as bool,
useElf: data["use_elf"] as bool,
useSdk: data["use_sdk"] as bool,
useHotReload: data["hot_reload"] as bool,
useHotReloadRollback: data["hot_reload_rollback"] as bool,
isHostChecked: data["host_checked"] as bool,
isCsp: data["csp"] as bool,
isMinified: data["minified"] as bool,
vmOptions: vmOptions,
dart2jsOptions: dart2jsOptions,
experiments: experiments,
babel: data['babel'] as String,
builderTag: data["builder_tag"] as String);
var configuration = TestConfiguration(
configuration: innerConfiguration,
progress: progress,
selectors: selectors,
testList: data["test_list_contents"] as List<String>,
repeat: data["repeat"] as int,
batch: !(data["noBatch"] as bool),
batchDart2JS: data["dart2js_batch"] as bool,
copyCoreDumps: data["copy_coredumps"] as bool,
isVerbose: data["verbose"] as bool,
listTests: data["list"] as bool,
listStatusFiles: data["list_status_files"] as bool,
cleanExit: data["clean_exit"] as bool,
silentFailures: data["silent_failures"] as bool,
printTiming: data["time"] as bool,
printReport: data["report"] as bool,
reportInJson: data["report_in_json"] as bool,
resetBrowser: data["reset_browser_configuration"] as bool,
skipCompilation: data["skip_compilation"] as bool,
writeDebugLog: data["write_debug_log"] as bool,
writeResults: data["write_results"] as bool,
writeLogs: data["write_logs"] as bool,
drtPath: data["drt"] as String,
chromePath: data["chrome"] as String,
safariPath: data["safari"] as String,
firefoxPath: data["firefox"] as String,
dartPath: data["dart"] as String,
dartPrecompiledPath: data["dart_precompiled"] as String,
genSnapshotPath: data["gen-snapshot"] as String,
keepGeneratedFiles: data["keep_generated_files"] as bool,
taskCount: data["tasks"] as int,
shardCount: data["shards"] as int,
shard: data["shard"] as int,
stepName: data["step_name"] as String,
testServerPort: data["test_server_port"] as int,
testServerCrossOriginPort:
data['test_server_cross_origin_port'] as int,
testDriverErrorPort: data["test_driver_error_port"] as int,
localIP: data["local_ip"] as String,
sharedOptions: sharedOptions,
packages: data["packages"] as String,
packageRoot: data["package_root"] as String,
suiteDirectory: data["suite_dir"] as String,
outputDirectory: data["output_directory"] as String,
reproducingArguments:
_reproducingCommand(data, namedConfiguration != null),
fastTestsOnly: data["fast_tests"] as bool,
printPassingStdout: data["print_passing_stdout"] as bool);
if (configuration.validate()) {
result.add(configuration);
}
var configuration = Configuration("custom configuration",
architecture, compiler, mode, runtime, system,
nnbdMode: nnbdMode,
timeout: data["timeout"] as int,
enableAsserts: data["enable_asserts"] as bool,
useAnalyzerCfe: data["use_cfe"] as bool,
useAnalyzerFastaParser:
data["analyzer_use_fasta_parser"] as bool,
useBlobs: data["use_blobs"] as bool,
useElf: data["use_elf"] as bool,
useSdk: data["use_sdk"] as bool,
useHotReload: data["hot_reload"] as bool,
useHotReloadRollback: data["hot_reload_rollback"] as bool,
isHostChecked: data["host_checked"] as bool,
isCsp: data["csp"] as bool,
isMinified: data["minified"] as bool,
vmOptions: vmOptions,
dart2jsOptions: dart2jsOptions,
experiments: experiments,
babel: data['babel'] as String,
builderTag: data["builder_tag"] as String);
addConfiguration(configuration);
}
}
}
@ -934,25 +957,6 @@ class OptionParseException implements Exception {
OptionParseException(this.message);
}
Configuration _namedConfiguration(String template) {
if (template == null) return null;
var testMatrixFile = "tools/bots/test_matrix.json";
var testMatrix = TestMatrix.fromPath(testMatrixFile);
var configuration = testMatrix.configurations
.singleWhere((c) => c.name == template, orElse: () => null);
if (configuration == null) {
var names = testMatrix.configurations
.map((configuration) => configuration.name)
.toList();
names.sort();
_fail('The named configuration "$template" does not exist. The following '
'configurations are available:\n * ${names.join('\n * ')}');
}
return configuration;
}
/// Throws an [OptionParseException] with [message].
void _fail(String message) {
throw OptionParseException(message);

3
pkg/test_runner/lib/src/test_configurations.dart
View file

@ -45,7 +45,6 @@ final TEST_SUITE_DIRECTORIES = [
Future testConfigurations(List<TestConfiguration> configurations) async {
var startTime = DateTime.now();
var startStopwatch = Stopwatch()..start();
// Extract global options from first configuration.
var firstConf = configurations[0];
@ -209,7 +208,7 @@ Future testConfigurations(List<TestConfiguration> configurations) async {
}
if (firstConf.writeResults) {
eventListener.add(ResultWriter(firstConf, startTime, startStopwatch));
eventListener.add(ResultWriter(firstConf.outputDirectory));
}
if (firstConf.copyCoreDumps) {

12
pkg/test_runner/lib/src/test_progress.dart
View file

@ -708,15 +708,11 @@ String _buildSummaryEnd(Formatter formatter, int failedTests) {
/// Writes a results.json file with a line for each test.
/// Each line is a json map with the test name and result and expected result.
class ResultWriter extends EventListener {
final TestConfiguration _configuration;
final List<Map> _results = [];
final List<Map> _logs = [];
final String _outputDirectory;
final Stopwatch _startStopwatch;
final DateTime _startTime;
ResultWriter(this._configuration, this._startTime, this._startStopwatch)
: _outputDirectory = _configuration.outputDirectory;
ResultWriter(this._outputDirectory);
void allTestsKnown() {
// Write an empty result log file, that will be overwritten if any tests
@ -729,10 +725,6 @@ class ResultWriter extends EventListener {
lines.map((l) => l + '\n').join();
void done(TestCase test) {
if (_configuration != test.configuration) {
throw Exception("Two configurations in the same run. "
"Cannot output results for multiple configurations.");
}
final name = test.displayName;
final index = name.indexOf('/');
final suite = name.substring(0, index);
@ -742,7 +734,7 @@ class ResultWriter extends EventListener {
final record = {
"name": name,
"configuration": _configuration.configuration.name,
"configuration": test.configuration.configuration.name,
"suite": suite,
"test_name": testName,
"time_ms": time.inMilliseconds,

284
tools/test.dart
View file

@ -6,7 +6,6 @@
// Run tests like on the given builder and/or named configuration.
import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io';
@ -136,16 +135,22 @@ String branchOfBuilder(String builder, List<String> branches) {
orElse: () => "master");
}
class ResolvedConfigurations {
final Set<String> configurationNames;
final Set<String> builders;
ResolvedConfigurations(this.configurationNames, this.builders);
}
/// Finds the named configuration to test according to the test matrix
/// information and the command line options.
bool resolveNamedConfiguration(
ResolvedConfigurations resolveNamedConfigurations(
List<String> branches,
List<dynamic> buildersConfigurations,
String requestedBranch,
String requestedNamedConfiguration,
String requestedBuilder,
Set<String> outputNamedConfiguration,
Set<String> outputBuilders) {
List<String> requestedNamedConfigurations,
String requestedBuilder) {
Set<String> namedConfigurations = {};
Set<String> builders = {};
bool foundBuilder = false;
for (final builderConfiguration in buildersConfigurations) {
for (final builder in builderConfiguration["builders"]) {
@ -160,7 +165,7 @@ bool resolveNamedConfiguration(
stderr.writeln("error: Builder $requestedBuilder is on branch $branch "
"rather than $requestedBranch");
stderr.writeln("error: To compare with that branch, use: -B $branch");
return false;
return null;
}
foundBuilder = true;
final steps = (builderConfiguration["steps"] as List).cast<Map>();
@ -172,50 +177,50 @@ bool resolveNamedConfiguration(
final arguments = step["arguments"]
.map((argument) => expandVariables(argument, builder))
.toList();
final namedConfiguration = arguments
final String namedConfiguration = arguments
.firstWhere((argument) => (argument as String).startsWith("-n"))
.substring(2);
if (requestedNamedConfiguration == null ||
requestedNamedConfiguration == namedConfiguration) {
outputNamedConfiguration.add(namedConfiguration);
outputBuilders.add(builder);
if (namedConfiguration.contains(",")) {
throw "Multiple named configurations in builder configurations: "
"are currently not supported: '$arguments'";
}
if (requestedNamedConfigurations.isEmpty ||
requestedNamedConfigurations.contains(namedConfiguration)) {
namedConfigurations.add(namedConfiguration);
builders.add(builder);
}
}
}
}
if (requestedBuilder != null && !foundBuilder) {
stderr.writeln("error: Builder $requestedBuilder doesn't exist");
return false;
return null;
}
if (requestedBuilder != null &&
requestedNamedConfiguration == null &&
outputNamedConfiguration.isEmpty) {
requestedNamedConfigurations == null &&
namedConfigurations.isEmpty) {
stderr.writeln("error: Builder $requestedBuilder isn't testing any named "
"configurations");
return false;
return null;
}
if (requestedBuilder != null &&
requestedNamedConfiguration != null &&
outputNamedConfiguration.isEmpty) {
requestedNamedConfigurations != null &&
namedConfigurations.isEmpty) {
stderr.writeln("error: The builder $requestedBuilder isn't testing the "
"named configuration $requestedNamedConfiguration");
return false;
"named configuration $requestedNamedConfigurations");
return null;
}
if (requestedNamedConfiguration != null && outputBuilders.isEmpty) {
if (requestedNamedConfigurations != null && builders.isEmpty) {
stderr.writeln("error: The named configuration "
"$requestedNamedConfiguration isn't tested on any builders");
return false;
"$requestedNamedConfigurations isn't tested on any builders");
return null;
}
return true;
return ResolvedConfigurations(namedConfigurations, builders);
}
/// Locates the merge base between head and the [branch] on the given [remote].
/// If a particular [commit] was requested, use that.
Future<String> findMergeBase(
String commit, String remote, String branch) async {
if (commit != null) {
return commit;
}
Future<String> findMergeBase(String remote, String branch) async {
final arguments = ["merge-base", "$remote/$branch", "HEAD"];
final result =
await Process.run("git", arguments, runInShell: Platform.isWindows);
@ -313,6 +318,42 @@ Future<BuildSearchResult> searchForApproximateBuild(
}
}
void overrideConfiguration(Map<String, Map<String, dynamic>> results,
String configuration, String newConfiguration) {
results.forEach((String key, Map<String, dynamic> result) {
if (result["configuration"] == configuration) {
result["configuration"] = newConfiguration;
}
});
}
void printUsage(ArgParser parser, {String error, bool printOptions: false}) {
if (error != null) {
print("$error\n");
exitCode = 1;
}
print("""
Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--]
[TEST.PY OPTION]... [SELECTOR]...
Run tests and compare with the results on the given builder. Either the -n or
the -b option, or both, must be used. Any options following -- and non-option
arguments will be forwarded to test.py invocations. The specified named
configuration's results will be downloaded from the specified builder. If only a
named configuration is specified, the results are downloaded from the
appropriate builders. If only a builder is specified, the default named
configuration is used if the builder only has a single named configuration.
Otherwise the available named configurations are listed.
See the documentation at https://goto.google.com/dart-status-file-free-workflow
""");
if (printOptions) {
print(parser.usage);
} else {
print("Run test.dart --help to see all options.");
}
}
void main(List<String> args) async {
final parser = new ArgParser();
parser.addOption("builder",
@ -330,9 +371,9 @@ void main(List<String> args) async {
"detected by --deflake will remain hidden");
parser.addFlag("list-configurations",
help: "Output list of configurations.", negatable: false);
parser.addOption("named-configuration",
parser.addMultiOption("named-configuration",
abbr: "n",
help: "The named test configuration that supplies the\nvalues for all "
help: "The named test configuration(s) that supplies the\nvalues for all "
"test options, specifying how tests\nshould be run.");
parser.addOption("local-configuration",
abbr: "N",
@ -346,27 +387,16 @@ void main(List<String> args) async {
defaultsTo: "origin");
parser.addFlag("help", help: "Show the program usage.", negatable: false);
final options = parser.parse(args);
if (options["help"] ||
(options["builder"] == null &&
options["named-configuration"] == null &&
!options["list-configurations"])) {
print("""
Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--]
[TEST.PY OPTION]... [SELECTOR]...
ArgResults options;
try {
options = parser.parse(args);
} on FormatException catch (exception) {
printUsage(parser, error: exception.message);
return;
}
Run tests and compare with the results on the given builder. Either the -n or
the -b option, or both, must be used. Any options following -- and non-option
arguments will be forwarded to test.py invocations. The specified named
configuration's results will be downloaded from the specified builder. If only a
named configuration is specified, the results are downloaded from the
appropriate builders. If only a builder is specified, the default named
configuration is used if the builder only has a single named configuration.
Otherwise the available named configurations are listed.
See the documentation at https://goto.google.com/dart-status-file-free-workflow
${parser.usage}""");
if (options["help"]) {
printUsage(parser, printOptions: true);
return;
}
@ -378,6 +408,25 @@ ${parser.usage}""");
return;
}
final requestedBuilder = options["builder"];
final namedConfigurations =
(options["named-configuration"] as List).cast<String>();
final localConfiguration = options["local-configuration"] as String;
if (requestedBuilder == null && namedConfigurations.isEmpty) {
printUsage(parser,
error: "Please specify either a configuration (-n) or "
"a builder (-b)");
return;
}
if (localConfiguration != null && namedConfigurations.length > 1) {
printUsage(parser,
error: "Local configuration (-N) can only be used with a"
" single named configuration (-n)");
return;
}
// Locate gsutil.py.
gsutilPy =
Platform.script.resolve("../third_party/gsutil/gsutil.py").toFilePath();
@ -394,47 +443,34 @@ ${parser.usage}""");
// Determine what named configuration to run and which builders to download
// existing results from.
final namedConfigurations = new SplayTreeSet<String>();
final builders = new SplayTreeSet<String>();
if (!resolveNamedConfiguration(
ResolvedConfigurations configurations = resolveNamedConfigurations(
branches,
buildersConfigurations,
options["branch"],
options["named-configuration"],
options["builder"],
namedConfigurations,
builders)) {
requestedBuilder);
if (configurations == null) {
// No valid configuration could be found. The error has already been
// reported by [resolveConfiguration].
exitCode = 1;
return;
}
if (namedConfigurations.length > 1) {
final builder = builders.single;
stderr.writeln(
"error: The builder $builder is testing multiple named configurations");
stderr.writeln(
"error: Please select the desired named configuration using -n:");
for (final namedConfiguration in namedConfigurations) {
stderr.writeln(" -n $namedConfiguration");
}
exitCode = 1;
return;
}
final namedConfiguration = namedConfigurations.single;
final localConfiguration =
options["local-configuration"] as String ?? namedConfiguration;
for (final builder in builders) {
if (localConfiguration != namedConfiguration) {
for (final builder in configurations.builders) {
if (localConfiguration != null) {
print("Testing the named configuration $localConfiguration "
"compared with builder $builder's configuration $namedConfiguration");
"compared with builder $builder's configuration "
"${namedConfigurations.single}");
} else {
print("Testing the named configuration $localConfiguration "
print("Testing the named configuration(s) "
"${namedConfigurations.join(",")} "
"compared with builder $builder");
}
}
// Find out where the current HEAD branched.
final commit = await findMergeBase(
options["commit"], options["remote"], options["branch"]);
// Use given commit or find out where the current HEAD branched.
final commit = options["commit"] ??
await findMergeBase(options["remote"], options["branch"]);
print("Base commit is $commit");
// Store the downloaded results and our test results in a temporary directory.
@ -443,9 +479,13 @@ ${parser.usage}""");
final mergedResults = <String, Map<String, dynamic>>{};
final mergedFlaky = <String, Map<String, dynamic>>{};
bool needsConfigurationOverride = localConfiguration != null &&
localConfiguration != namedConfigurations.single;
bool needsMerge = configurations.builders.length > 1;
// Use the buildbucket API to search for builds of the right commit.
final inexactBuilds = new SplayTreeMap<String, String>();
for (final builder in builders) {
final inexactBuilds = <String, String>{};
for (final builder in configurations.builders) {
// Download the previous results and flakiness info from cloud storage.
print("Finding build on builder $builder to compare with...");
final buildSearchResult =
@ -463,64 +503,63 @@ ${parser.usage}""");
await cpGsutil(buildFileCloudPath(builder, buildNumber, "flaky.json"),
"${outDirectory.path}/flaky.json");
}
print("Downloaded baseline results from builder $builder");
// Merge the results for the builders.
if (builders.length > 1) {
mergedResults
.addAll(await loadResultsMap("${outDirectory.path}/previous.json"));
if (needsMerge || needsConfigurationOverride) {
var results =
await loadResultsMap("${outDirectory.path}/previous.json");
if (needsConfigurationOverride) {
overrideConfiguration(
results, namedConfigurations.single, localConfiguration);
}
mergedResults.addAll(results);
if (!options["report-flakes"]) {
mergedFlaky
.addAll(await loadResultsMap("${outDirectory.path}/flaky.json"));
var flakyTests =
await loadResultsMap("${outDirectory.path}/flaky.json");
if (needsConfigurationOverride) {
overrideConfiguration(
flakyTests, namedConfigurations.single, localConfiguration);
}
mergedFlaky.addAll(flakyTests);
}
}
}
// Write out the merged results for the builders.
if (builders.length > 1) {
if (needsMerge || needsConfigurationOverride) {
await new File("${outDirectory.path}/previous.json").writeAsString(
mergedResults.values.map((data) => jsonEncode(data) + "\n").join(""));
}
// Ensure that there is a flaky.json even if it wasn't downloaded.
if (builders.length > 1 || options["report-flakes"]) {
if (needsMerge || needsConfigurationOverride || options["report-flakes"]) {
await new File("${outDirectory.path}/flaky.json").writeAsString(
mergedFlaky.values.map((data) => jsonEncode(data) + "\n").join(""));
}
// Override the named configuration in the baseline data if needed.
if (namedConfiguration != localConfiguration) {
for (final path in [
"${outDirectory.path}/previous.json",
"${outDirectory.path}/flaky.json"
]) {
final results = await loadResultsMap(path);
final records = results.values
.where((r) => r["configuration"] == namedConfiguration)
.toList()
..forEach((r) => r["configuration"] = localConfiguration);
await new File(path).writeAsString(
records.map((data) => jsonEncode(data) + "\n").join(""));
}
}
final configurationsToRun = localConfiguration != null
? <String>[localConfiguration]
: namedConfigurations;
// Run the tests.
final arguments = [
"--named-configuration=$localConfiguration",
"--output-directory=${outDirectory.path}",
"--clean-exit",
"--silent-failures",
"--write-results",
"--write-logs",
...options.rest,
];
print("".padLeft(80, "="));
print("Running tests");
print("".padLeft(80, "="));
await runProcessInheritStdio("python", ["tools/test.py", ...arguments],
await runProcessInheritStdio(
"python",
[
"tools/test.py",
"--named-configuration=${configurationsToRun.join(",")}",
"--output-directory=${outDirectory.path}",
"--clean-exit",
"--silent-failures",
"--write-results",
"--write-logs",
...options.rest,
],
runInShell: Platform.isWindows);
if (options["deflake"]) {
await deflake(outDirectory, localConfiguration, options.rest);
await deflake(outDirectory, configurationsToRun, options.rest);
}
// Write out the final comparison.
@ -546,16 +585,19 @@ ${parser.usage}""");
}
if (inexactBuilds.isNotEmpty) {
print("");
inexactBuilds.forEach((String builder, String inexactCommit) => print(
"Warning: Results may be inexact because commit ${inexactCommit} "
"was used as the baseline for $builder instead of $commit"));
final builders = inexactBuilds.keys.toList()..sort();
for (var builder in builders) {
final inexactCommit = inexactBuilds[builder];
print("Warning: Results may be inexact because commit ${inexactCommit} "
"was used as the baseline for $builder instead of $commit");
}
}
} finally {
await outDirectory.delete(recursive: true);
}
}
void deflake(Directory outDirectory, String localConfiguration,
void deflake(Directory outDirectory, List<String> configurations,
List<String> testPyArgs) async {
// Find the list of tests to deflake.
final deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
@ -580,7 +622,7 @@ void deflake(Directory outDirectory, String localConfiguration,
final deflakeDirectory = new Directory("${outDirectory.path}/$i");
await deflakeDirectory.create();
final deflakeArguments = <String>[
"--named-configuration=$localConfiguration",
"--named-configuration=${configurations.join(",")}",
"--output-directory=${deflakeDirectory.path}",
"--clean-exit",
"--silent-failures",