mirror of
https://github.com/dart-lang/sdk
synced 2024-10-03 09:10:10 +00:00
9394b30b77
Bug: b/286184681 Change-Id: I903528c4adfbc576644aec7541903df6b9633e26 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/325523 Reviewed-by: Jonas Termansen <sortie@google.com> Reviewed-by: Samuel Rawlins <srawlins@google.com> Commit-Queue: Alexander Thomas <athom@google.com>
675 lines
22 KiB
Dart
675 lines
22 KiB
Dart
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
|
|
import 'dart:async' show Timer;
|
|
import 'dart:convert' show jsonEncode;
|
|
import 'dart:io' show File, Platform, exitCode;
|
|
import 'dart:isolate' show Isolate, ReceivePort, SendPort;
|
|
|
|
import 'package:args/args.dart' show ArgParser;
|
|
import 'package:testing/src/chain.dart' show CreateContext, Result, Step;
|
|
import 'package:testing/src/expectation.dart' show Expectation;
|
|
import 'package:testing/src/log.dart' show Logger;
|
|
import 'package:testing/src/run.dart' show runMe;
|
|
import 'package:testing/src/suite.dart' as testing show Suite;
|
|
import 'package:testing/src/test_description.dart' show TestDescription;
|
|
|
|
import 'dartdoctest_suite.dart' as dartdoctest show createContext;
|
|
import 'fasta/expression_suite.dart' as expression show createContext;
|
|
import 'fasta/incremental_dartino_suite.dart' as incremental_dartino
|
|
show createContext;
|
|
import 'fasta/messages_suite.dart' as messages show createContext;
|
|
import 'fasta/modular_suite.dart' as modular show createContext;
|
|
import 'fasta/outline_suite.dart' as outline show createContext;
|
|
import 'fasta/strong_suite.dart' as strong show createContext;
|
|
import 'fasta/textual_outline_suite.dart' as textual_outline show createContext;
|
|
import 'fasta/weak_suite.dart' as weak show createContext;
|
|
import 'incremental_bulk_compiler_smoke_suite.dart' as incremental_bulk_compiler
|
|
show createContext;
|
|
import 'incremental_suite.dart' as incremental show createContext;
|
|
import 'lint_suite.dart' as lint show createContext;
|
|
import 'outline_extractor_suite.dart' as outline_extractor show createContext;
|
|
import 'parser_all_suite.dart' as parserAll show createContext;
|
|
import 'parser_equivalence_suite.dart' as parserEquivalence show createContext;
|
|
import 'parser_suite.dart' as parser show createContext;
|
|
import 'spelling_test_not_src_suite.dart' as spelling_not_src
|
|
show createContext;
|
|
import 'spelling_test_src_suite.dart' as spelling_src show createContext;
|
|
|
|
const suiteNamePrefix = "pkg/front_end/test";
|
|
|
|
int getDefaultThreads() {
|
|
int numberOfWorkers = 1;
|
|
if (Platform.numberOfProcessors > 2) {
|
|
numberOfWorkers = Platform.numberOfProcessors - 1;
|
|
}
|
|
return numberOfWorkers;
|
|
}
|
|
|
|
class Options {
|
|
final String? configurationName;
|
|
final bool verbose;
|
|
final bool printFailureLog;
|
|
final Uri outputDirectory;
|
|
final String? testFilter;
|
|
final List<String> environmentOptions;
|
|
final int shardCount;
|
|
final int shard;
|
|
final bool skipTestsThatRequireGit;
|
|
final bool onlyTestsThatRequireGit;
|
|
final int numberOfWorkers;
|
|
|
|
Options(
|
|
this.configurationName,
|
|
this.verbose,
|
|
this.printFailureLog,
|
|
this.outputDirectory,
|
|
this.testFilter,
|
|
this.environmentOptions, {
|
|
required this.shardCount,
|
|
required this.shard,
|
|
required this.skipTestsThatRequireGit,
|
|
required this.onlyTestsThatRequireGit,
|
|
required this.numberOfWorkers,
|
|
});
|
|
|
|
static Options parse(List<String> args) {
|
|
var parser = new ArgParser()
|
|
..addOption("named-configuration",
|
|
abbr: "n",
|
|
help: "configuration name to use for emitting json result files")
|
|
..addOption("output-directory",
|
|
help: "directory to which results.json and logs.json are written")
|
|
..addFlag("verbose",
|
|
abbr: "v", help: "print additional information", defaultsTo: false)
|
|
..addFlag("print",
|
|
abbr: "p", help: "print failure logs", defaultsTo: false)
|
|
..addMultiOption('environment',
|
|
abbr: 'D', help: "environment options for the test suite")
|
|
..addOption("tasks",
|
|
abbr: "j",
|
|
help: "The number of parallel tasks to run.",
|
|
defaultsTo: "${getDefaultThreads()}")
|
|
..addOption("shards", help: "Number of shards", defaultsTo: "1")
|
|
..addOption("shard", help: "Which shard to run", defaultsTo: "1")
|
|
..addFlag("skipTestsThatRequireGit",
|
|
help: "Whether to skip tests that require git to run",
|
|
defaultsTo: false)
|
|
..addFlag("onlyTestsThatRequireGit",
|
|
help: "Whether to only run tests that require git",
|
|
defaultsTo: false);
|
|
var parsedOptions = parser.parse(args);
|
|
String outputPath = parsedOptions["output-directory"] ?? ".";
|
|
Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath));
|
|
|
|
bool verbose = parsedOptions["verbose"];
|
|
|
|
String? filter;
|
|
if (parsedOptions.rest.length == 1) {
|
|
filter = parsedOptions.rest.single;
|
|
if (filter.startsWith("$suiteNamePrefix/")) {
|
|
filter = filter.substring(suiteNamePrefix.length + 1);
|
|
}
|
|
}
|
|
String tasksString = parsedOptions["tasks"];
|
|
int? tasks = int.tryParse(tasksString);
|
|
if (tasks == null || tasks < 1) {
|
|
throw "--tasks (-j) has to be an integer >= 1";
|
|
}
|
|
|
|
String shardsString = parsedOptions["shards"];
|
|
int? shardCount = int.tryParse(shardsString);
|
|
if (shardCount == null || shardCount < 1) {
|
|
throw "--shards has to be an integer >= 1";
|
|
}
|
|
String shardString = parsedOptions["shard"];
|
|
int? shard = int.tryParse(shardString);
|
|
if (shard == null || shard < 1 || shard > shardCount) {
|
|
throw "--shard has to be an integer >= 1 (and <= --shards)";
|
|
}
|
|
bool skipTestsThatRequireGit = parsedOptions["skipTestsThatRequireGit"];
|
|
bool onlyTestsThatRequireGit = parsedOptions["onlyTestsThatRequireGit"];
|
|
if (skipTestsThatRequireGit && onlyTestsThatRequireGit) {
|
|
throw "Only one of --skipTestsThatRequireGit and "
|
|
"--onlyTestsThatRequireGit can be provided.";
|
|
}
|
|
|
|
if (verbose) {
|
|
print("NOTE: Created with options\n "
|
|
"${parsedOptions["named-configuration"]},\n "
|
|
"${verbose},\n "
|
|
"${parsedOptions["print"]},\n "
|
|
"${outputDirectory},\n "
|
|
"${filter},\n "
|
|
"${parsedOptions['environment']},\n "
|
|
"shardCount: ${shardCount},\n "
|
|
"shard: ${shard - 1 /* make it 0-indexed */},\n "
|
|
"onlyTestsThatRequireGit: ${onlyTestsThatRequireGit},\n "
|
|
"skipTestsThatRequireGit: ${skipTestsThatRequireGit},\n "
|
|
"numberOfWorkers: ${tasks}");
|
|
}
|
|
|
|
return Options(
|
|
parsedOptions["named-configuration"],
|
|
verbose,
|
|
parsedOptions["print"],
|
|
outputDirectory,
|
|
filter,
|
|
parsedOptions['environment'],
|
|
shardCount: shardCount,
|
|
shard: shard - 1 /* make it 0-indexed */,
|
|
onlyTestsThatRequireGit: onlyTestsThatRequireGit,
|
|
skipTestsThatRequireGit: skipTestsThatRequireGit,
|
|
numberOfWorkers: tasks,
|
|
);
|
|
}
|
|
}
|
|
|
|
class ResultLogger implements Logger {
|
|
final String prefix;
|
|
final bool verbose;
|
|
final bool printFailureLog;
|
|
final SendPort resultsPort;
|
|
final SendPort logsPort;
|
|
final Map<String, Stopwatch> stopwatches = {};
|
|
final String? configurationName;
|
|
final Set<String> seenTests = {};
|
|
bool gotFrameworkError = false;
|
|
|
|
ResultLogger(this.prefix, this.resultsPort, this.logsPort, this.verbose,
|
|
this.printFailureLog, this.configurationName);
|
|
|
|
String getTestName(TestDescription description) {
|
|
return "$prefix/${description.shortName}";
|
|
}
|
|
|
|
@override
|
|
void logMessage(Object message) {}
|
|
|
|
@override
|
|
void logNumberedLines(String text) {}
|
|
|
|
@override
|
|
void logProgress(String message) {}
|
|
|
|
@override
|
|
void logStepComplete(int completed, int failed, int total,
|
|
testing.Suite suite, TestDescription description, Step step) {}
|
|
|
|
@override
|
|
void logStepStart(int completed, int failed, int total, testing.Suite suite,
|
|
TestDescription description, Step step) {}
|
|
|
|
@override
|
|
void logSuiteStarted(testing.Suite suite) {}
|
|
|
|
@override
|
|
void logSuiteComplete(testing.Suite suite) {}
|
|
|
|
void handleTestResult(
|
|
testing.Suite suite,
|
|
TestDescription testDescription,
|
|
Result result,
|
|
String fullSuiteName,
|
|
bool matchedExpectations,
|
|
Set<Expectation> expectedOutcomes) {
|
|
String testName = getTestName(testDescription);
|
|
String suiteName = "pkg";
|
|
String shortTestName = testName.substring(suiteName.length + 1);
|
|
resultsPort.send(jsonEncode({
|
|
"name": testName,
|
|
"configuration": configurationName,
|
|
"suite": suiteName,
|
|
"test_name": shortTestName,
|
|
"time_ms": stopwatches[testName]!.elapsedMilliseconds,
|
|
"expected": "Pass",
|
|
"result": matchedExpectations ? "Pass" : "Fail",
|
|
"matches": matchedExpectations,
|
|
}));
|
|
if (!matchedExpectations) {
|
|
StringBuffer sb = new StringBuffer();
|
|
if (printFailureLog) {
|
|
String outcome = "${result.outcome}";
|
|
sb.write("FAILED: $testName: $outcome");
|
|
}
|
|
sb.write(result.log);
|
|
if (result.error != null) {
|
|
sb.write("\n\n${result.error}");
|
|
}
|
|
if (result.trace != null) {
|
|
sb.write("\n\n${result.trace}");
|
|
}
|
|
sb.write("\n\nTo re-run this test, run:");
|
|
var extraFlags = _assertsEnabled ? ' --enable-asserts' : '';
|
|
sb.write(
|
|
"\n\n dart$extraFlags pkg/front_end/test/unit_test_suites.dart -p "
|
|
"$testName");
|
|
if (result.autoFixCommand != null) {
|
|
sb.write("\n\nTo automatically update the test expectations, run:");
|
|
sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p "
|
|
"$testName -D${result.autoFixCommand}");
|
|
if (result.canBeFixWithUpdateExpectations) {
|
|
sb.write('\n\nTo update test expectations for all tests at once, '
|
|
'run:');
|
|
sb.write('\n\n dart pkg/front_end/tool/update_expectations.dart');
|
|
sb.write('\n\nNote that this takes a long time and should only be '
|
|
'used when many tests need updating.\n');
|
|
}
|
|
}
|
|
if (result.outcome == Expectation.pass) {
|
|
String expectedString =
|
|
expectedOutcomes.map((e) => e.toString()).join(", ");
|
|
sb.write("\n\nThe test passed, but wasn't expected to. "
|
|
"You should update the status file for this test."
|
|
"\nThere's a status entry looking something like"
|
|
"\n\n ${testDescription.shortName}: ${expectedString}"
|
|
"\n\nwhich should be removed."
|
|
"\n\nThe status file is ${suite.statusFile}.");
|
|
} else if (result.autoFixCommand == null) {
|
|
String expectedString =
|
|
expectedOutcomes.map((e) => e.toString()).join(", ");
|
|
sb.write("\n\nThe test has outcome ${result.outcome}, "
|
|
"but was expected to have outcome(s) ${expectedOutcomes}. "
|
|
"You might have to update the status file to the new outcome"
|
|
"\nThere's a status entry looking something like"
|
|
"\n\n ${testDescription.shortName}: ${expectedString}"
|
|
"\n\nwhich should be updated."
|
|
"\n\nThe status file is ${suite.statusFile}.");
|
|
}
|
|
String failureLog = sb.toString();
|
|
String outcome = "${result.outcome}";
|
|
logsPort.send(jsonEncode({
|
|
"name": testName,
|
|
"configuration": configurationName,
|
|
"result": outcome,
|
|
"log": failureLog,
|
|
}));
|
|
if (printFailureLog) {
|
|
print(failureLog);
|
|
}
|
|
}
|
|
if (verbose) {
|
|
String result = matchedExpectations ? "PASS" : "FAIL";
|
|
print("${testName}: ${result}");
|
|
}
|
|
}
|
|
|
|
@override
|
|
void logTestStart(int completed, int failed, int total, testing.Suite suite,
|
|
TestDescription description) {
|
|
String name = getTestName(description);
|
|
stopwatches[name] = Stopwatch()..start();
|
|
}
|
|
|
|
@override
|
|
void logTestComplete(int completed, int failed, int total,
|
|
testing.Suite suite, TestDescription description) {}
|
|
|
|
@override
|
|
void logUncaughtError(error, StackTrace stackTrace) {}
|
|
|
|
@override
|
|
void logExpectedResult(testing.Suite suite, TestDescription description,
|
|
Result result, Set<Expectation> expectedOutcomes) {
|
|
handleTestResult(
|
|
suite, description, result, prefix, true, expectedOutcomes);
|
|
}
|
|
|
|
@override
|
|
void logUnexpectedResult(testing.Suite suite, TestDescription description,
|
|
Result result, Set<Expectation> expectedOutcomes) {
|
|
// The test framework (pkg/testing) calls the logger with an unexpected
|
|
// results a second time to create a summary. We ignore the second call
|
|
// here.
|
|
String testName = getTestName(description);
|
|
if (seenTests.contains(testName)) return;
|
|
seenTests.add(testName);
|
|
handleTestResult(
|
|
suite, description, result, prefix, false, expectedOutcomes);
|
|
}
|
|
|
|
@override
|
|
void noticeFrameworkCatchError(error, StackTrace stackTrace) {
|
|
gotFrameworkError = true;
|
|
}
|
|
}
|
|
|
|
class Suite {
|
|
final CreateContext createContext;
|
|
final String testingRootPath;
|
|
final String? path;
|
|
final int shardCount;
|
|
final String prefix;
|
|
final bool requiresGit;
|
|
|
|
const Suite(
|
|
this.prefix,
|
|
this.createContext,
|
|
this.testingRootPath, {
|
|
required this.shardCount,
|
|
this.path,
|
|
this.requiresGit = false,
|
|
});
|
|
}
|
|
|
|
const List<Suite> suites = [
|
|
const Suite(
|
|
"dartdoctest",
|
|
dartdoctest.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"fasta/expression",
|
|
expression.createContext,
|
|
"../../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"fasta/outline",
|
|
outline.createContext,
|
|
"../../testing.json",
|
|
shardCount: 2,
|
|
),
|
|
const Suite(
|
|
"fasta/incremental_dartino",
|
|
incremental_dartino.createContext,
|
|
"../../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"fasta/messages",
|
|
messages.createContext,
|
|
"../../testing.json",
|
|
shardCount: 1,
|
|
requiresGit: true,
|
|
),
|
|
const Suite(
|
|
"fasta/strong",
|
|
strong.createContext,
|
|
"../../testing.json",
|
|
path: "fasta/strong_suite.dart",
|
|
shardCount: 2,
|
|
),
|
|
const Suite(
|
|
"incremental_bulk_compiler_smoke",
|
|
incremental_bulk_compiler.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"incremental",
|
|
incremental.createContext,
|
|
"../testing.json",
|
|
shardCount: 2,
|
|
),
|
|
const Suite(
|
|
"lint",
|
|
lint.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
requiresGit: true,
|
|
),
|
|
const Suite(
|
|
"parser",
|
|
parser.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"parser_equivalence",
|
|
parserEquivalence.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"parser_all",
|
|
parserAll.createContext,
|
|
"../testing.json",
|
|
shardCount: 4,
|
|
requiresGit:
|
|
true /* technically not true, but tests *many* more files
|
|
than in test_matrix.json file set */
|
|
,
|
|
),
|
|
const Suite(
|
|
"spelling_test_not_src",
|
|
spelling_not_src.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
requiresGit: true,
|
|
),
|
|
const Suite(
|
|
"spelling_test_src",
|
|
spelling_src.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
requiresGit: true,
|
|
),
|
|
const Suite(
|
|
"fasta/modular",
|
|
modular.createContext,
|
|
"../../testing.json",
|
|
path: "fasta/modular_suite.dart",
|
|
shardCount: 4,
|
|
),
|
|
const Suite(
|
|
"fasta/weak",
|
|
weak.createContext,
|
|
"../../testing.json",
|
|
path: "fasta/weak_suite.dart",
|
|
shardCount: 10,
|
|
),
|
|
const Suite(
|
|
"fasta/textual_outline",
|
|
textual_outline.createContext,
|
|
"../../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
const Suite(
|
|
"outline_extractor",
|
|
outline_extractor.createContext,
|
|
"../testing.json",
|
|
shardCount: 1,
|
|
),
|
|
];
|
|
|
|
const Duration timeoutDuration = Duration(minutes: 30);
|
|
|
|
class SuiteConfiguration {
|
|
final Suite suite;
|
|
final SendPort resultsPort;
|
|
final SendPort logsPort;
|
|
final bool verbose;
|
|
final bool printFailureLog;
|
|
final String? configurationName;
|
|
final String? testFilter;
|
|
final List<String> environmentOptions;
|
|
final int shard;
|
|
|
|
const SuiteConfiguration(
|
|
this.suite,
|
|
this.resultsPort,
|
|
this.logsPort,
|
|
this.verbose,
|
|
this.printFailureLog,
|
|
this.configurationName,
|
|
this.testFilter,
|
|
this.environmentOptions,
|
|
this.shard,
|
|
);
|
|
}
|
|
|
|
Future<void> runSuite(SuiteConfiguration configuration) async {
|
|
Suite suite = configuration.suite;
|
|
String name = suite.prefix;
|
|
String fullSuiteName = "$suiteNamePrefix/$name";
|
|
Uri suiteUri = Platform.script.resolve(suite.path ?? "${name}_suite.dart");
|
|
if (!new File.fromUri(suiteUri).existsSync()) {
|
|
throw "File doesn't exist: $suiteUri";
|
|
}
|
|
ResultLogger logger = ResultLogger(
|
|
fullSuiteName,
|
|
configuration.resultsPort,
|
|
configuration.logsPort,
|
|
configuration.verbose,
|
|
configuration.printFailureLog,
|
|
configuration.configurationName);
|
|
await runMe(
|
|
<String>[
|
|
if (configuration.testFilter != null) configuration.testFilter!,
|
|
for (String option in configuration.environmentOptions) '-D${option}',
|
|
],
|
|
suite.createContext,
|
|
me: suiteUri,
|
|
configurationPath: suite.testingRootPath,
|
|
logger: logger,
|
|
shards: suite.shardCount,
|
|
shard: configuration.shard,
|
|
);
|
|
if (logger.gotFrameworkError) {
|
|
throw "Got framework error!";
|
|
}
|
|
}
|
|
|
|
Future<void> writeLinesToFile(Uri uri, List<String> lines) async {
|
|
await File.fromUri(uri).writeAsString(lines.map((line) => "$line\n").join());
|
|
}
|
|
|
|
Future<void> main([List<String> arguments = const <String>[]]) async {
|
|
Stopwatch totalRuntime = new Stopwatch()..start();
|
|
|
|
List<String> results = [];
|
|
List<String> logs = [];
|
|
Options options = Options.parse(arguments);
|
|
ReceivePort resultsPort = new ReceivePort()
|
|
..listen((resultEntry) => results.add(resultEntry));
|
|
ReceivePort logsPort = new ReceivePort()
|
|
..listen((logEntry) => logs.add(logEntry));
|
|
List<Future<bool>> futures = [];
|
|
|
|
if (options.verbose) {
|
|
print("NOTE: Willing to run with ${options.numberOfWorkers} 'workers'");
|
|
print("");
|
|
}
|
|
|
|
int numberOfFreeWorkers = options.numberOfWorkers;
|
|
// Run test suites and record the results and possible failure logs.
|
|
int chunkNum = 0;
|
|
for (Suite suite in suites) {
|
|
if (options.onlyTestsThatRequireGit && !suite.requiresGit) continue;
|
|
if (options.skipTestsThatRequireGit && suite.requiresGit) continue;
|
|
String prefix = suite.prefix;
|
|
String? filter = options.testFilter;
|
|
if (filter != null) {
|
|
// Skip suites that are not hit by the test filter, is there is one.
|
|
if (!filter.startsWith(prefix)) {
|
|
continue;
|
|
}
|
|
// Remove the 'fasta/' from filters, if there, because it is not used
|
|
// in the name defined in testing.json.
|
|
if (filter.startsWith("fasta/")) {
|
|
filter = filter.substring("fasta/".length);
|
|
}
|
|
}
|
|
for (int shard = 0; shard < suite.shardCount; shard++) {
|
|
if (chunkNum++ % options.shardCount != options.shard) continue;
|
|
|
|
while (numberOfFreeWorkers <= 0) {
|
|
// This might not be great design, but it'll work fine.
|
|
await Future.delayed(const Duration(milliseconds: 50));
|
|
}
|
|
numberOfFreeWorkers--;
|
|
// Start the test suite in a new isolate.
|
|
ReceivePort exitPort = new ReceivePort();
|
|
ReceivePort errorPort = new ReceivePort();
|
|
SuiteConfiguration configuration = new SuiteConfiguration(
|
|
suite,
|
|
resultsPort.sendPort,
|
|
logsPort.sendPort,
|
|
options.verbose,
|
|
options.printFailureLog,
|
|
options.configurationName,
|
|
filter,
|
|
options.environmentOptions,
|
|
shard);
|
|
Future<bool> future = new Future<bool>(() async {
|
|
try {
|
|
Stopwatch stopwatch = new Stopwatch()..start();
|
|
String naming = "$prefix";
|
|
if (suite.shardCount > 1) {
|
|
naming += " (${shard + 1} of ${suite.shardCount})";
|
|
}
|
|
print("Running suite $naming");
|
|
Isolate isolate = await Isolate.spawn<SuiteConfiguration>(
|
|
runSuite, configuration,
|
|
onExit: exitPort.sendPort, onError: errorPort.sendPort);
|
|
bool timedOutOrCrash = false;
|
|
Timer timer = new Timer(timeoutDuration, () {
|
|
timedOutOrCrash = true;
|
|
print("Suite $naming timed out after "
|
|
"${timeoutDuration.inMilliseconds}ms");
|
|
isolate.kill(priority: Isolate.immediate);
|
|
});
|
|
await exitPort.first;
|
|
errorPort.close();
|
|
List<dynamic> allErrors = await errorPort.toList();
|
|
bool gotError = allErrors.isNotEmpty;
|
|
if (gotError) {
|
|
print("Suite $naming encountered ${allErrors.length} error(s).");
|
|
print("Errors:");
|
|
for (int i = 0; i < allErrors.length; i++) {
|
|
print("-----------");
|
|
print("Error #$i:");
|
|
print(allErrors[i]);
|
|
}
|
|
print("-----------");
|
|
timedOutOrCrash = true;
|
|
}
|
|
timer.cancel();
|
|
int seconds = stopwatch.elapsedMilliseconds ~/ 1000;
|
|
if (!timedOutOrCrash) {
|
|
print("Suite $naming finished (took ${seconds} seconds)");
|
|
} else {
|
|
print("Suite $naming finished badly (see above) "
|
|
"(took ${seconds} seconds)");
|
|
}
|
|
return timedOutOrCrash;
|
|
} finally {
|
|
numberOfFreeWorkers++;
|
|
}
|
|
});
|
|
futures.add(future);
|
|
}
|
|
}
|
|
// Wait for isolates to terminate and clean up.
|
|
Iterable<bool> timeoutsOrCrashes = await Future.wait(futures);
|
|
resultsPort.close();
|
|
logsPort.close();
|
|
// Write results.json and logs.json.
|
|
Uri resultJsonUri = options.outputDirectory.resolve("results.json");
|
|
Uri logsJsonUri = options.outputDirectory.resolve("logs.json");
|
|
await writeLinesToFile(resultJsonUri, results);
|
|
await writeLinesToFile(logsJsonUri, logs);
|
|
print("Log files written to ${resultJsonUri.toFilePath()} and"
|
|
" ${logsJsonUri.toFilePath()}");
|
|
print("Entire run took ${totalRuntime.elapsed}.");
|
|
// Return with exit code 1 if at least one suite timed out.
|
|
bool timedOutOrCrashed = timeoutsOrCrashes.any((timeout) => timeout);
|
|
if (timedOutOrCrashed) {
|
|
throw "Crashed or timed out. Check stdout for more details.";
|
|
} else {
|
|
// The testing framework (package:testing) sets the exitCode to `1` if any
|
|
// test failed, so we reset it here to indicate that the test runner was
|
|
// successful.
|
|
exitCode = 0;
|
|
}
|
|
}
|
|
|
|
final bool _assertsEnabled = () {
|
|
bool assertsEnabled = false;
|
|
assert(assertsEnabled = true);
|
|
return assertsEnabled;
|
|
}();
|