languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-10-14 09:31:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add filtering to "--list-configurations".

Browse source 
I got tired of wading through the giant list and guessing at which one
matches the options I want to run the tests on, so I added filtering.
If you pass any of the common options like "-m", "-r", etc. when also
passing "--list-configurations", then it only prints configurations
that match those options.

Also, by default it only prints configurations that match the current
host OS.

Eventually, I would like *running* tests to work the same way, where
passing "-c" means "find me a config in the test matrix with this
compiler. But this seems like a good start.

Also, I removed the slow way that test.dart calls test.py to handle
--list-configurations now that those are all in the same package.

Change-Id: Ifabb415a9fad889afc12cfcd7dd81bd02c918612
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/158980
Commit-Queue: Bob Nystrom <rnystrom@google.com>
Auto-Submit: Bob Nystrom <rnystrom@google.com>
Reviewed-by: Karl Klose <karlklose@google.com>
This commit is contained in:
Robert Nystrom 2020-08-20 19:05:49 +00:00 committed by commit-bot@chromium.org
parent d11e89f516
commit 7ec29a9b6d
2 changed files with 133 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
pkg/test_runner/lib/src/options.dart
View file

@ -155,7 +155,7 @@ simarm, simarmv6, simarm64, arm_x64''',
hide: true),
_Option('system', 'The operating system to run tests on.',
abbr: 's',
values: System.names,
values: ['all', ...System.names],
defaultsTo: Platform.operatingSystem,
hide: true),
_Option('sanitizer', 'Sanitizer in which to run the tests.',
@ -228,6 +228,7 @@ compact, color, line, verbose, silent, status, buildbot''',
_Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.',
hide: true),
_Option.bool('list', 'List tests only, do not run them.'),
_Option.bool('find-configurations', 'Find matching configurations.'),
_Option.bool('list-configurations', 'Output list of configurations.'),
_Option.bool('list_status_files',
'List status files for test-suites. Do not run any test suites.',
@ -406,21 +407,8 @@ compiler.''',
return null;
}
if (arguments.contains("--list-configurations")) {
var testMatrixFile = "tools/bots/test_matrix.json";
var testMatrix = TestMatrix.fromPath(testMatrixFile);
for (var configuration in testMatrix.configurations
.map((configuration) => configuration.name)
.toList()
..sort()) {
print(configuration);
}
return null;
}
var configuration = <String, dynamic>{};
// Fill in configuration with arguments passed to the test script.
// Parse the command line arguments to a map.
var options = <String, dynamic>{};
for (var i = 0; i < arguments.length; i++) {
var arg = arguments[i];
@ -458,7 +446,7 @@ compiler.''',
} else {
// The argument does not start with "-" or "--" and is therefore not an
// option. Use it as a test selector pattern.
var patterns = configuration.putIfAbsent("selectors", () => <String>[]);
var patterns = options.putIfAbsent("selectors", () => <String>[]);
// Allow passing in the full relative path to a test or directory and
// infer the selector from it. This lets users use tab completion on
@ -492,7 +480,7 @@ compiler.''',
// Multiple uses of a flag are an error, because there is no naturally
// correct way to handle conflicting options.
if (configuration.containsKey(option.name)) {
if (options.containsKey(option.name)) {
_fail('Already have value for command line option "$command".');
}
@ -503,12 +491,12 @@ compiler.''',
_fail('Boolean flag "$command" does not take a value.');
}
configuration[option.name] = true;
options[option.name] = true;
break;
case _OptionValueType.int:
try {
configuration[option.name] = int.parse(value);
options[option.name] = int.parse(value);
} on FormatException {
_fail('Integer value expected for option "$command".');
}
@ -535,17 +523,27 @@ compiler.''',
// TODO(rnystrom): Store as a list instead of a comma-delimited
// string.
configuration[option.name] = value;
options[option.name] = value;
break;
}
}
if (options.containsKey('find-configurations')) {
findConfigurations(options);
return null;
}
if (options.containsKey('list-configurations')) {
listConfigurations(options);
return null;
}
// If a named configuration was specified ensure no other options, which are
// implied by the named configuration, were specified.
if (configuration['named_configuration'] is String) {
if (options['named_configuration'] is String) {
for (var optionName in _namedConfigurationOptions) {
if (configuration.containsKey(optionName)) {
var namedConfig = configuration['named_configuration'];
if (options.containsKey(optionName)) {
var namedConfig = options['named_configuration'];
_fail("The named configuration '$namedConfig' implies "
"'$optionName'. Try removing '$optionName'.");
}
@ -554,26 +552,26 @@ compiler.''',
// Apply default values for unspecified options.
for (var option in _options) {
if (!configuration.containsKey(option.name)) {
configuration[option.name] = option.defaultValue;
if (!options.containsKey(option.name)) {
options[option.name] = option.defaultValue;
}
}
// Fetch list of tests to run, if option is present.
var testList = configuration['test_list'];
var testList = options['test_list'];
if (testList is String) {
configuration['test_list_contents'] = File(testList).readAsLinesSync();
options['test_list_contents'] = File(testList).readAsLinesSync();
}
var tests = configuration['tests'];
var tests = options['tests'];
if (tests is String) {
if (configuration.containsKey('test_list_contents')) {
if (options.containsKey('test_list_contents')) {
_fail('--tests and --test-list cannot be used together');
}
configuration['test_list_contents'] = LineSplitter.split(tests).toList();
options['test_list_contents'] = LineSplitter.split(tests).toList();
}
return _createConfigurations(configuration);
return _createConfigurations(options);
}
/// Given a set of parsed option values, returns the list of command line
@ -682,6 +680,12 @@ compiler.''',
data['progress'] = 'verbose';
}
var systemName = data["system"] as String;
if (systemName == "all") {
_fail("Can only use '--system=all' with '--find-configurations'.");
}
var system = System.find(systemName);
var runtimeNames = data["runtime"] as String;
var runtimes = [
if (runtimeNames != null) ...runtimeNames.split(",").map(Runtime.find)
@ -813,7 +817,6 @@ compiler.''',
}
for (var sanitizerName in sanitizers.split(",")) {
var sanitizer = Sanitizer.find(sanitizerName);
var system = System.find(data["system"] as String);
var configuration = Configuration("custom configuration",
architecture, compiler, mode, runtime, system,
nnbdMode: nnbdMode,
@ -996,6 +999,101 @@ class OptionParseException implements Exception {
OptionParseException(this.message);
}
/// Prints the names of the configurations in the test matrix that match the
/// given filter options.
///
/// If any of the options `--system`, `--arch`, `--mode`, `--compiler`,
/// `--nnbd`, or `--runtime` (or their abbreviations) are passed, then only
/// configurations matching those are shown.
void findConfigurations(Map<String, dynamic> options) {
var testMatrix = TestMatrix.fromPath('tools/bots/test_matrix.json');
// Default to only showing configurations for the current machine.
var systemOption = options['system'] as String;
var system = System.host;
if (systemOption == 'all') {
system = null;
} else if (systemOption != null) {
system = System.find(systemOption);
}
var architectureOption = options['arch'] as String;
var architectures = const [Architecture.x64];
if (architectureOption == 'all') {
architectures = null;
} else if (architectureOption != null) {
architectures =
architectureOption.split(',').map(Architecture.find).toList();
}
var mode = Mode.release;
if (options.containsKey('mode')) {
mode = Mode.find(options['mode'] as String);
}
Compiler compiler;
if (options.containsKey('compiler')) {
compiler = Compiler.find(options['compiler'] as String);
}
Runtime runtime;
if (options.containsKey('runtime')) {
runtime = Runtime.find(options['runtime'] as String);
}
NnbdMode nnbdMode;
if (options.containsKey('nnbd')) {
nnbdMode = NnbdMode.find(options['nnbd'] as String);
}
var names = <String>[];
for (var configuration in testMatrix.configurations) {
if (system != null && configuration.system != system) continue;
if (architectures != null &&
!architectures.contains(configuration.architecture)) {
continue;
}
if (mode != null && configuration.mode != mode) continue;
if (compiler != null && configuration.compiler != compiler) continue;
if (runtime != null && configuration.runtime != runtime) continue;
if (nnbdMode != null && configuration.nnbdMode != nnbdMode) continue;
names.add(configuration.name);
}
names.sort();
var filters = [
if (system != null) "system=$system",
if (architectures != null) "arch=${architectures.join(',')}",
if (mode != null) "mode=$mode",
if (compiler != null) "compiler=$compiler",
if (runtime != null) "runtime=$runtime",
if (nnbdMode != null) "nnbd=$nnbdMode",
];
if (filters.isEmpty) {
print("All configurations:");
} else {
print("Configurations where ${filters.join(', ')}:");
}
for (var name in names) {
print("- $name");
}
}
/// Prints the names of the configurations in the test matrix.
void listConfigurations(Map<String, dynamic> options) {
var testMatrix = TestMatrix.fromPath('tools/bots/test_matrix.json');
var names = testMatrix.configurations
.map((configuration) => configuration.name)
.toList();
names.sort();
names.forEach(print);
}
/// Throws an [OptionParseException] with [message].
void _fail(String message) {
throw OptionParseException(message);

6
pkg/test_runner/lib/test_runner.dart
View file

@ -12,6 +12,7 @@ import 'package:args/args.dart';
import 'package:smith/smith.dart';
import 'bot_results.dart';
import 'src/options.dart';
const int deflakingCount = 5;
@ -343,10 +344,7 @@ Future<void> runTests(List<String> args) async {
}
if (options["list-configurations"] as bool) {
var process = await Process.start(
"python", ["tools/test.py", "--list-configurations"],
mode: ProcessStartMode.inheritStdio, runInShell: Platform.isWindows);
exitCode = await process.exitCode;
listConfigurations({"system": "all"});
return;
}