diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index f18405b8255..d324552eb9e 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -156,6 +156,9 @@ dart/optimized_stacktrace_line_and_column_test: RuntimeError # minifying they can be renamed, which is issue 7953. dart/inline_stack_frame_test: RuntimeError, Pass # Issue 7953 +[ $compiler == dart2js && $cps_ir && $checked ] +dart/*: Skip # checked mode + cpsir not supported yet. Issue 25761 + [ $compiler == dart2js || $compiler == dart2analyzer ] # Data uri's not supported by dart2js or the analyzer. dart/data_uri*test: Skip diff --git a/samples/samples.status b/samples/samples.status index 2c735a8064c..4a9105ebddd 100644 --- a/samples/samples.status +++ b/samples/samples.status @@ -14,6 +14,9 @@ sample_extension: Skip [ $compiler == dart2js && $runtime == none ] *: Fail, Pass # TODO(ahe): Triage these tests. +[ $compiler == dart2js && $cps_ir && $checked ] +sample_extension: Crash # Unable to compile UnsupportedError.message. + [ $compiler == dart2analyzer ] build_dart: Skip diff --git a/tools/testing/dart/co19_test.dart b/tools/testing/dart/co19_test.dart index 29113a07ccf..5cbcddaf36b 100644 --- a/tools/testing/dart/co19_test.dart +++ b/tools/testing/dart/co19_test.dart @@ -18,7 +18,7 @@ library co19_test; import "dart:io"; -import "options.dart"; +import "test_options.dart"; import "test_suite.dart"; import "test_configurations.dart"; @@ -59,7 +59,7 @@ const List> COMMAND_LINES = const >[ void main(List args) { TestUtils.setDartDirUri(Platform.script.resolve('../../..')); - var optionsParser = new OptionsParser(); + var optionsParser = new TestOptionsParser(); var configurations = []; for (var commandLine in COMMAND_LINES) { var arguments = []; diff --git a/tools/testing/dart/compiler_configuration.dart b/tools/testing/dart/compiler_configuration.dart index aa6486bbb73..fffd68135bc 100644 --- a/tools/testing/dart/compiler_configuration.dart +++ b/tools/testing/dart/compiler_configuration.dart @@ -47,12 +47,6 @@ abstract class CompilerConfiguration { final bool isHostChecked; final bool useSdk; - /// Only some subclasses support this check, but we statically allow calling - /// it on [CompilerConfiguration]. - bool get useDfe { - throw new UnsupportedError("This compiler does not support DFE."); - } - // TODO(ahe): Remove this constructor and move the switch to // test_options.dart. We probably want to store an instance of // [CompilerConfiguration] in [configuration] there. @@ -68,6 +62,7 @@ abstract class CompilerConfiguration { bool isHostChecked = configuration['host_checked']; bool useSdk = configuration['use_sdk']; bool isCsp = configuration['csp']; + bool useCps = configuration['cps_ir']; bool useBlobs = configuration['use_blobs']; bool hotReload = configuration['hot_reload']; bool hotReloadRollback = configuration['hot_reload_rollback']; @@ -87,6 +82,7 @@ abstract class CompilerConfiguration { isDebug: isDebug, isChecked: isChecked, isHostChecked: isHostChecked, + useCps: useCps, useSdk: useSdk, isCsp: isCsp, useFastStartup: useFastStartup, @@ -111,7 +107,7 @@ abstract class CompilerConfiguration { useSdk: useSdk, hotReload: hotReload, hotReloadRollback: hotReloadRollback, - useDfe: true); + useDFE: true); case 'dartkp': return new PrecompilerCompilerConfiguration( isDebug: isDebug, @@ -119,7 +115,7 @@ abstract class CompilerConfiguration { arch: configuration['arch'], useBlobs: useBlobs, isAndroid: configuration['system'] == 'android', - useDfe: true); + useDFE: true); case 'none': return new NoneCompilerConfiguration( isDebug: isDebug, @@ -191,7 +187,7 @@ abstract class CompilerConfiguration { class NoneCompilerConfiguration extends CompilerConfiguration { final bool hotReload; final bool hotReloadRollback; - final bool useDfe; + final bool useDFE; NoneCompilerConfiguration( {bool isDebug, @@ -200,7 +196,7 @@ class NoneCompilerConfiguration extends CompilerConfiguration { bool useSdk, bool this.hotReload, bool this.hotReloadRollback, - this.useDfe: false}) + this.useDFE: false}) : super._subclass( isDebug: isDebug, isChecked: isChecked, @@ -218,7 +214,7 @@ class NoneCompilerConfiguration extends CompilerConfiguration { List originalArguments, CommandArtifact artifact) { List args = []; - if (useDfe) { + if (useDFE) { args.add('--dfe=${buildDir}/gen/kernel-service.dart.snapshot'); args.add('--platform=${buildDir}/patched_sdk/platform.dill'); } @@ -541,15 +537,20 @@ class Dart2xCompilerConfiguration extends CompilerConfiguration { /// Configuration for dart2js compiler. class Dart2jsCompilerConfiguration extends Dart2xCompilerConfiguration { final bool isCsp; + final bool useCps; final bool useFastStartup; final bool useKernel; final List extraDart2jsOptions; + // We cache the extended environment to save memory. + static Map cpsFlagCache; + static Map environmentOverridesCacheObject; Dart2jsCompilerConfiguration( {bool isDebug, bool isChecked, bool isHostChecked, bool useSdk, + bool this.useCps, bool this.isCsp, bool this.useFastStartup, this.useKernel, @@ -603,7 +604,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { final String arch; final bool useBlobs; final bool isAndroid; - final bool useDfe; + final bool useDFE; PrecompilerCompilerConfiguration( {bool isDebug, @@ -611,7 +612,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { this.arch, this.useBlobs, this.isAndroid, - this.useDfe: false}) + this.useDFE: false}) : super._subclass(isDebug: isDebug, isChecked: isChecked); int computeTimeoutMultiplier() { @@ -657,7 +658,7 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration { exec = "$buildDir/dart_bootstrap"; } var args = []; - if (useDfe) { + if (useDFE) { args.add('--dfe=utils/kernel-service/kernel-service.dart'); args.add('--platform=${buildDir}/patched_sdk/platform.dill'); } diff --git a/tools/testing/dart/main.dart b/tools/testing/dart/main.dart index ff2a0d12641..727cccf18b5 100644 --- a/tools/testing/dart/main.dart +++ b/tools/testing/dart/main.dart @@ -22,8 +22,8 @@ /// `factory StandardTestSuite.forDirectory`. import "dart:io"; -import "options.dart"; import "test_configurations.dart"; +import "test_options.dart"; import "test_suite.dart"; /// Runs all of the tests specified by the given command line [arguments]. @@ -32,8 +32,8 @@ void main(List arguments) { TestUtils.setDartDirUri(Platform.script.resolve('../../..')); // Parse the command line arguments to a configuration. - var parser = new OptionsParser(); - var configurations = parser.parse(arguments); + var optionsParser = new TestOptionsParser(); + var configurations = optionsParser.parse(arguments); if (configurations == null || configurations.isEmpty) return; // Run all of the configured tests. diff --git a/tools/testing/dart/options.dart b/tools/testing/dart/options.dart deleted file mode 100644 index 685e7c93df3..00000000000 --- a/tools/testing/dart/options.dart +++ /dev/null @@ -1,916 +0,0 @@ -// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:io'; -import 'drt_updater.dart'; -import 'test_suite.dart'; -import 'path.dart'; -import 'compiler_configuration.dart' show CompilerConfiguration; -import 'runtime_configuration.dart' show RuntimeConfiguration; - -const _defaultTestSelectors = const [ - 'samples', - 'standalone', - 'corelib', - 'co19', - 'language', - 'isolate', - 'vm', - 'html', - 'benchmark_smoke', - 'utils', - 'lib', - 'analyze_library', - 'service', - 'kernel', - 'observatory_ui' -]; - -/// Specifies a single command line option. -/// -/// The name of the specification is used as the key for the option in the Map -/// returned from the [TestOptionParser] parse method. -class _Option { - // TODO(rnystrom): Some string options use "" to mean "no value" and others - // use null. Clean that up. - _Option(this.name, this.description, - {String abbr, List values, String defaultsTo = ""}) - : abbreviation = abbr, - values = values ?? [], - defaultValue = defaultsTo, - type = _OptionValueType.string; - - _Option.bool(this.name, this.description, [this.abbreviation]) - : values = [], - defaultValue = false, - type = _OptionValueType.bool; - - _Option.int(this.name, this.description, {String abbr, int defaultsTo}) - : abbreviation = abbr, - values = [], - defaultValue = defaultsTo, - type = _OptionValueType.int; - - final String name; - final String description; - final String abbreviation; - final List values; - final Object defaultValue; - final _OptionValueType type; - - /// Gets the shortest command line argument used to refer to this option. - String get shortCommand => abbreviation != null ? "-$abbreviation" : command; - - /// Gets the canonical long command line argument used to refer to this - /// option. - String get command => "--${name.replaceAll('_', '-')}"; -} - -enum _OptionValueType { bool, int, string } - -/// Parses command line arguments and produces a test runner configuration. -class OptionsParser { - static final List<_Option> _options = [ - new _Option('mode', 'Mode in which to run the tests.', - abbr: 'm', - values: ['all', 'debug', 'release', 'product'], - defaultsTo: 'debug'), - new _Option( - 'compiler', - '''Specify any compilation step (if needed). - -none: Do not compile the Dart code (run native Dart code - on the VM). - (Only valid with runtimes vm, flutter, or drt.) - -precompiler: Compile into AOT snapshot before running the test. - (Only valid with runtime dart_precompiled.) - -dart2js: Compile dart code to JavaScript by running dart2js. - (Only valid with runtimes: d8, drt, chrome, safari, - ie9, ie10, ie11, firefox, opera, chromeOnAndroid, - and none [compile only].) - -dart2analyzer: Perform static analysis on Dart code by running the - analyzer. - (Only valid with runtime none.) - -app_jit: Compile the Dart code into an app snapshot before - running test. - (Only valid with dart_app runtime.) - -dartk: Compile the Dart source into Kernel before running - test. - -dartkp: Compile the Dart source into Kernel and then Kernel - into AOT snapshot before running the test. - (Only valid with runtime dart_precompiled.)''', - abbr: 'c', - values: [ - 'none', - 'precompiler', - 'dart2js', - 'dart2analyzer', - 'app_jit', - 'dartk', - 'dartkp' - ], - defaultsTo: 'none'), - new _Option( - 'runtime', - '''Where the tests should be run. -vm: Run Dart code on the standalone dart vm. - -flutter: Run Dart code on the flutter engine. - -dart_precompiled: Run a precompiled snapshot on a variant of the - standalone dart VM lacking a JIT. - -d8: Run JavaScript from the command line using v8. - -jsshell: Run JavaScript from the command line using - Firefox js-shell. - -drt: Run Dart or JavaScript in the headless version - of Chrome, Content shell. - -dartium: Run Dart or JavaScript in Dartium. - -ContentShellOnAndroid: Run Dart or JavaScript in Dartium content - shell on Android. - -DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. - -ff: -chrome: -safari: -ie9: -ie10: -ie11: -opera: -chromeOnAndroid: Run JavaScript in the specified browser. - -self_check: Pass each test or its compiled output to every - file under `pkg` whose name ends with - `_self_check.dart`. Each test is given to the - self_check tester as a filename on stdin using - the batch-mode protocol. - -none: No runtime, compile only. (For example, used - for dart2analyzer static analysis tests).''', - abbr: 'r', - values: [ - 'vm', - 'flutter', - 'dart_precompiled', - 'd8', - 'jsshell', - 'drt', - 'dartium', - 'ff', - 'firefox', - 'chrome', - 'safari', - 'ie9', - 'ie10', - 'ie11', - 'opera', - 'chromeOnAndroid', - 'safarimobilesim', - 'ContentShellOnAndroid', - 'DartiumOnAndroid', - 'self_check', - 'none' - ], - defaultsTo: 'vm'), - new _Option( - 'arch', - '''The architecture to run tests for. - -Allowed values are: -all -ia32, x64 -arm, armv6, armv5te, arm64, -simarm, simarmv6, simarmv5te, simarm64, -mips, simmips -simdbc, simdbc64''', - abbr: 'a', - values: [ - 'all', - 'ia32', - 'x64', - 'arm', - 'armv6', - 'armv5te', - 'arm64', - 'mips', - 'simarm', - 'simarmv6', - 'simarmv5te', - 'simarm64', - 'simmips', - 'simdbc', - 'simdbc64', - ], - defaultsTo: 'x64'), - new _Option('system', 'The operating system to run tests on.', - abbr: 's', - values: ['linux', 'macos', 'windows', 'android'], - defaultsTo: Platform.operatingSystem), - new _Option.bool('checked', 'Run tests in checked mode.'), - new _Option.bool('strong', 'Run tests in strong mode.'), - new _Option.bool('host_checked', 'Run compiler in checked mode.'), - new _Option.bool('minified', 'Enable minification in the compiler.'), - new _Option.bool( - 'csp', 'Run tests under Content Security Policy restrictions.'), - new _Option.bool( - 'fast_startup', 'Pass the --fast-startup flag to dart2js.'), - new _Option.bool('dart2js_with_kernel', - 'Enable the internal pipeline in dart2js to use kernel.'), - new _Option.bool('hot_reload', 'Run hot reload stress tests.'), - new _Option.bool( - 'hot_reload_rollback', 'Run hot reload rollback stress tests.'), - new _Option.bool('use_blobs', - 'Use mmap instead of shared libraries for precompilation.'), - new _Option.int('timeout', 'Timeout in seconds.', - abbr: 't', defaultsTo: -1), - new _Option( - 'progress', - '''Progress indication mode. - -Allowed values are: -compact, color, line, verbose, silent, status, buildbot, diff -''', - abbr: 'p', - values: [ - 'compact', - 'color', - 'line', - 'verbose', - 'silent', - 'status', - 'buildbot', - 'diff' - ], - defaultsTo: 'compact'), - new _Option('step_name', 'Step name for use by -pbuildbot.', - defaultsTo: null), - new _Option.bool('report', - 'Print a summary report of the number of tests, by expectation.'), - new _Option.int('tasks', 'The number of parallel tasks to run.', - abbr: 'j', defaultsTo: Platform.numberOfProcessors), - new _Option.int('shards', - 'The number of instances that the tests will be sharded over.', - defaultsTo: 1), - new _Option.int( - 'shard', 'The index of this instance when running in sharded mode.', - defaultsTo: 1), - new _Option.bool('help', 'Print list of options.', 'h'), - new _Option.bool('verbose', 'Verbose output.', 'v'), - new _Option.bool('verify-ir', 'Verify kernel IR.'), - new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.'), - new _Option.bool('list', 'List tests only, do not run them.'), - new _Option.bool('report_in_json', - 'When listing with --list, output result summary in JSON.'), - new _Option.bool('time', 'Print timing information after running tests.'), - new _Option('dart', 'Path to dart executable.'), - new _Option('flutter', 'Path to flutter executable.'), - new _Option( - 'drt', // TODO(antonm): fix the option name. - 'Path to content shell executable.'), - new _Option('dartium', 'Path to Dartium Chrome executable.'), - new _Option('firefox', 'Path to firefox browser executable.'), - new _Option('chrome', 'Path to chrome browser executable.'), - new _Option('safari', 'Path to safari browser executable.'), - new _Option.bool( - 'use_sdk', - '''Use compiler or runtime from the SDK. - -Normally, the compiler or runtimes in PRODUCT_DIR is tested, with -this option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin -is tested. - -(Note: currently only implemented for dart2js.)'''), - new _Option('build_directory', - 'The name of the build directory, where products are placed.'), - new _Option.bool('noBatch', 'Do not run tests in batch mode.', 'n'), - new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.'), - new _Option.bool( - 'append_logs', 'Do not delete old logs but rather append to them.'), - new _Option.bool('write_debug_log', - 'Don\'t write debug messages to stdout but rather to a logfile.'), - new _Option.bool('write_test_outcome_log', - 'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.'), - new _Option.bool( - 'reset_browser_configuration', - '''Browser specific reset of configuration. - -Warning: Using this option may remove your bookmarks and other -settings.'''), - new _Option.bool( - 'copy_coredumps', - '''If we see a crash that we did not expect, copy the core dumps to -"/tmp".'''), - new _Option( - 'local_ip', - '''IP address the HTTP servers should listen on. This address is also -used for browsers to connect to.''', - defaultsTo: '127.0.0.1'), - new _Option.int('test_server_port', 'Port for test http server.', - defaultsTo: 0), - new _Option.int('test_server_cross_origin_port', - 'Port for test http server cross origin.', - defaultsTo: 0), - new _Option.int('test_driver_port', 'Port for http test driver server.', - defaultsTo: 0), - new _Option.int( - 'test_driver_error_port', 'Port for http test driver server errors.', - defaultsTo: 0), - new _Option('record_to_file', - 'Records all commands to be executed and writes to a file.', - defaultsTo: null), - new _Option( - 'replay_from_file', 'Replays a previously recorded list of commands.', - defaultsTo: null), - new _Option( - 'builder_tag', - '''Machine specific options that is not captured by the regular test -options. Used to be able to make sane updates to the status files.'''), - new _Option('vm_options', 'Extra options to send to the vm when running.', - defaultsTo: null), - new _Option( - 'dart2js_options', 'Extra options for dart2js compilation step.', - defaultsTo: null), - new _Option( - 'suite_dir', 'Additional directory to add to the testing matrix.', - defaultsTo: null), - new _Option('package_root', 'The package root to use for testing.', - defaultsTo: null), - new _Option('packages', 'The package spec file to use for testing.', - defaultsTo: null), - new _Option( - 'exclude_suite', - '''Exclude suites from default selector, only works when no selector -has been specified on the command line.''', - defaultsTo: null), - new _Option.bool( - 'skip-compilation', - ''' -Skip the compilation step, using the compilation artifacts left in -the output folder from a previous run. This flag will often cause -false positves and negatives, but can be useful for quick and -dirty offline testing when not making changes that affect the -compiler.''') - ]; - - /// For printing out reproducing command lines, we don't want to add these - /// options. - static final _blacklistedOptions = [ - 'append_logs', - 'build_directory', - 'chrome', - 'copy_coredumps', - 'dart', - 'flutter', - 'dartium', - 'drt', - 'exclude_suite', - 'firefox', - 'local_ip', - 'progress', - 'report', - 'safari', - 'shard', - 'shards', - 'step_name', - 'tasks', - 'time', - 'verbose', - 'write_debug_log', - 'write_test_outcome_log', - ].toSet(); - - /// Parses a list of strings as test options. - /// - /// Returns a list of configurations in which to run the tests. - /// Configurations are maps mapping from option keys to values. When - /// encountering the first non-option string, the rest of the arguments are - /// stored in the returned Map under the 'rest' key. - List parse(List arguments) { - // TODO(rnystrom): The builders on the buildbots still pass this even - // though it does nothing. Until those can be fixed, silently ignore the - // option. Remove this once the buildbot scripts are fixed. - if (arguments.contains("--failure-summary")) { - arguments = arguments.where((arg) => arg != "--failure-summary").toList(); - print('Note: Ignoring unsupported "--failure-summary" option.'); - } - - var configuration = {}; - - // Fill in configuration with arguments passed to the test script. - for (var i = 0; i < arguments.length; i++) { - var arg = arguments[i]; - - // Help supersedes all other arguments. - if (arg == "--help" || arg == "-h") { - _printHelp(); - return null; - } - - // Extract name and value for options. - String command; - String value; - _Option option; - - if (arg.startsWith("--")) { - // A long option name. - var equals = arg.indexOf("="); - if (equals != -1) { - // A long option with a value, like "--arch=ia32". - command = arg.substring(0, equals); - value = arg.substring(equals + 1); - } else { - command = arg; - } - - option = _findByName(command.substring(2)); - } else if (arg.startsWith("-")) { - // An abbreviated option. - if (arg.length == 1) { - _fail('Missing option name after "-".'); - } - - command = arg.substring(0, 2); - - if (arg.length > 2) { - // An abbreviated option followed by a value, like "-aia32". - value = arg.substring(2); - } - - option = _findByAbbreviation(command.substring(1)); - } else { - // The argument does not start with "-" or "--" and is therefore not an - // option. Use it as a test selector pattern. - var patterns = configuration.putIfAbsent("selectors", () => []); - patterns.add(arg); - continue; - } - - if (option == null) { - _fail('Unknown command line option "$command".'); - } - - // If we need a value, look at the next argument. - if (value == null && option.type != _OptionValueType.bool) { - if (i + 1 >= arguments.length) { - _fail('Missing value for command line option "$command".'); - } - value = arguments[++i]; - } - - // Multiple uses of a flag are an error, because there is no naturally - // correct way to handle conflicting options. - if (configuration.containsKey(option.name)) { - _fail('Already have value for command line option "$command".'); - } - - // Parse the value for the option. - switch (option.type) { - case _OptionValueType.bool: - if (value != null) { - _fail('Boolean flag "$command" does not take a value.'); - } - - configuration[option.name] = true; - break; - - case _OptionValueType.int: - try { - configuration[option.name] = int.parse(value); - } on FormatException { - _fail('Integer value expected for option "$command".'); - } - break; - - case _OptionValueType.string: - // Validate against the allowed values. - if (!option.values.isEmpty) { - for (var v in value.split(",")) { - if (!option.values.contains(v)) { - _fail('Unknown value "$v" for command line option "$command".'); - } - } - } - - // TODO(rnystrom): Store as a list instead of a comma-delimited - // string. - configuration[option.name] = value; - break; - } - } - - // Apply default values for unspecified options. - for (var option in _options) { - if (!configuration.containsKey(option.name)) { - configuration[option.name] = option.defaultValue; - } - } - - var expandedConfigs = _expandConfigurations(configuration); - var result = expandedConfigs.where(_isValidConfig).toList(); - for (var config in result) { - config['_reproducing_arguments_'] = _reproducingCommand(config); - } - - return result.isEmpty ? null : result; - } - - /// Prints [message] and exits with a non-zero exit code. - void _fail(String message) { - print(message); - exit(1); - } - - /// Given a configuration, returns the list of command line arguments that - /// would reproduce that configuration. - List _reproducingCommand(Map config) { - var arguments = []; - - for (var option in _options) { - var name = option.name; - if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { - continue; - } - - var value = config[name]; - if (config[name] == option.defaultValue || - (name == 'packages' && - value == - TestUtils.dartDirUri.resolve('.packages').toFilePath())) { - continue; - } - - arguments.add(option.shortCommand); - if (option.type != _OptionValueType.bool) { - arguments.add(value.toString()); - } - } - - return arguments; - } - - /// Determines if a particular configuration has a valid combination of - /// compiler and runtime elements. - bool _isValidConfig(Map config) { - var isValid = true; - List validRuntimes; - switch (config['compiler']) { - case 'dart2js': - // Note: by adding 'none' as a configuration, if the user - // runs test.py -c dart2js -r drt,none the dart2js_none and - // dart2js_drt will be duplicating work. If later we don't need 'none' - // with dart2js, we should remove it from here. - validRuntimes = const [ - 'd8', - 'jsshell', - 'drt', - 'none', - 'dartium', - 'ff', - 'chrome', - 'safari', - 'ie9', - 'ie10', - 'ie11', - 'opera', - 'chromeOnAndroid', - 'safarimobilesim' - ]; - break; - case 'dart2analyzer': - validRuntimes = const ['none']; - break; - case 'app_jit': - case 'dartk': - validRuntimes = const ['vm', 'self_check', 'none']; - break; - case 'precompiler': - case 'dartkp': - validRuntimes = const ['dart_precompiled']; - break; - case 'none': - validRuntimes = const [ - 'vm', - 'flutter', - 'drt', - 'dartium', - 'ContentShellOnAndroid', - 'DartiumOnAndroid' - ]; - break; - } - - if (!validRuntimes.contains(config['runtime'])) { - isValid = false; - print("Warning: combination of compiler '${config['compiler']}' and " - "runtime '${config['runtime']}' is invalid. " - "Skipping this combination."); - } - - if (config['ie'] && Platform.operatingSystem != 'windows') { - isValid = false; - print("Warning: cannot run Internet Explorer on non-Windows operating" - " system."); - } - - if (config['shard'] < 1 || config['shard'] > config['shards']) { - isValid = false; - print("Error: shard index is ${config['shard']} out of " - "${config['shards']} shards"); - } - - if (config['runtime'] == 'flutter' && config['flutter'] == '') { - isValid = false; - print("-rflutter requires the flutter engine executable to " - "be specified using --flutter="); - } - - if (config['runtime'] == 'flutter' && config['arch'] != 'x64') { - isValid = false; - print("-rflutter is applicable only for --arch=x64"); - } - - return isValid; - } - - /// Recursively expands a configuration with multiple values per key into a - /// list of configurations with exactly one value per key. - List _expandConfigurations(Map configuration) { - // Expand the pseudo-values such as 'all'. - if (configuration['arch'] == 'all') { - configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; - } - - if (configuration['mode'] == 'all') { - configuration['mode'] = 'debug,release,product'; - } - - if (configuration['report_in_json']) { - configuration['list'] = true; - configuration['report'] = true; - } - - // Use verbose progress indication for verbose output unless buildbot - // progress indication is requested. - if (configuration['verbose'] && configuration['progress'] != 'buildbot') { - configuration['progress'] = 'verbose'; - } - - // Create the artificial negative options that test status files - // expect. - configuration['unchecked'] = !configuration['checked']; - configuration['host_unchecked'] = !configuration['host_checked']; - configuration['unminified'] = !configuration['minified']; - configuration['nocsp'] = !configuration['csp']; - - String runtime = configuration['runtime']; - if (runtime == 'firefox') { - configuration['runtime'] == 'ff'; - } - - String compiler = configuration['compiler']; - configuration['browser'] = TestUtils.isBrowserRuntime(runtime); - configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); - - // Set the javascript command line flag for less verbose status files. - configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); - - // Allow suppression that is valid for all ie versions - configuration['ie'] = runtime.startsWith('ie'); - - // Expand the test selectors into a suite name and a simple - // regular expressions to be used on the full path of a test file - // in that test suite. If no selectors are explicitly given use - // the default suite patterns. - var selectors = configuration['selectors']; - if (selectors is! Map) { - if (selectors == null) { - if (configuration['suite_dir'] != null) { - var suite_path = new Path(configuration['suite_dir']); - selectors = [suite_path.filename]; - } else { - selectors = _defaultTestSelectors.toList(); - } - - var excludeSuites = configuration['exclude_suite'] != null - ? configuration['exclude_suite'].split(',') - : []; - for (var exclude in excludeSuites) { - if (selectors.contains(exclude)) { - selectors.remove(exclude); - } else { - print("Warning: default selectors does not contain $exclude"); - } - } - } - var selectorMap = {}; - for (var i = 0; i < selectors.length; i++) { - var pattern = selectors[i]; - var suite = pattern; - var slashLocation = pattern.indexOf('/'); - if (slashLocation != -1) { - suite = pattern.substring(0, slashLocation); - pattern = pattern.substring(slashLocation + 1); - pattern = pattern.replaceAll('*', '.*'); - } else { - pattern = ".?"; - } - if (selectorMap.containsKey(suite)) { - print("Error: '$suite/$pattern'. Only one test selection" - " pattern is allowed to start with '$suite/'"); - exit(1); - } - selectorMap[suite] = new RegExp(pattern); - } - configuration['selectors'] = selectorMap; - } - - // Put observatory_ui in a configuration with its own packages override. - // Only one value in the configuration map is mutable: - selectors = configuration['selectors']; - if (selectors.containsKey('observatory_ui')) { - if (selectors.length == 1) { - configuration['packages'] = TestUtils.dartDirUri - .resolve('runtime/observatory/.packages') - .toFilePath(); - } else { - // Make a new configuration whose selectors map only contains - // observatory_ui, and remove the key from the original selectors. - // The only mutable value in the map is the selectors, so a - // shallow copy is safe. - var observatoryConfiguration = new Map.from(configuration); - observatoryConfiguration['selectors'] = { - 'observatory_ui': selectors['observatory_ui'] - }; - selectors.remove('observatory_ui'); - - // Set the packages flag. - observatoryConfiguration['packages'] = TestUtils.dartDirUri - .resolve('runtime/observatory/.packages') - .toFilePath(); - - // Return the expansions of both configurations. Neither will reach - // this line in the recursive call to _expandConfigurations. - return _expandConfigurations(configuration) - ..addAll(_expandConfigurations(observatoryConfiguration)); - } - } - - // Set the default package spec explicitly. - if (configuration['package_root'] == null && - configuration['packages'] == null) { - configuration['packages'] = - TestUtils.dartDirUri.resolve('.packages').toFilePath(); - } - - // Expand the architectures. - if (configuration['arch'].contains(',')) { - return _expandHelper('arch', configuration); - } - - // Expand modes. - if (configuration['mode'].contains(',')) { - return _expandHelper('mode', configuration); - } - - // Expand compilers. - if (configuration['compiler'].contains(',')) { - return _expandHelper('compiler', configuration); - } - - // Expand runtimes. - var runtimes = configuration['runtime']; - if (runtimes.contains(',')) { - return _expandHelper('runtime', configuration); - } else { - // All runtimes eventually go through this path, after expansion. - var updater = runtimeUpdater(configuration); - if (updater != null) { - updater.update(); - } - } - - // Adjust default timeout based on mode, compiler, and sometimes runtime. - if (configuration['timeout'] == -1) { - var isReload = - configuration['hot_reload'] || configuration['hot_reload_rollback']; - int compilerMulitiplier = - new CompilerConfiguration(configuration).computeTimeoutMultiplier(); - int runtimeMultiplier = new RuntimeConfiguration(configuration) - .computeTimeoutMultiplier( - mode: configuration['mode'], - isChecked: configuration['checked'], - isReload: isReload, - arch: configuration['arch']); - configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; - } - - return [configuration]; - } - - /// Helper for _expandConfigurations. Creates a new configuration and adds it - /// to a list, for use in a case when a particular configuration has multiple - /// results (separated by a ','). - /// Arguments: - /// option: The particular test option we are expanding. - /// configuration: The map containing all test configuration information - /// specified. - List _expandHelper(String option, Map configuration) { - var result = []; - var configs = configuration[option]; - for (var config in configs.split(',')) { - var newConfiguration = new Map.from(configuration); - newConfiguration[option] = config; - result.addAll(_expandConfigurations(newConfiguration)); - } - return result; - } - - /// Print out usage information. - void _printHelp() { - var buffer = new StringBuffer(); - - buffer.writeln('''usage: dart test.dart [options] [selector] - -The optional selector limits the tests that will be run. -For example, the selector "language/issue", or equivalently -"language/*issue*", limits to test files matching the regexp -".*issue.*\\.dart" in the "tests/language" directory. - -Options:'''); - - for (var option in _options) { - if (option.abbreviation != null) { - buffer.write("-${option.abbreviation}, "); - } else { - buffer.write(" "); - } - - buffer.write(option.command); - - switch (option.type) { - case _OptionValueType.bool: - // No value. - break; - case _OptionValueType.int: - buffer.write("="); - break; - case _OptionValueType.string: - if (option.values.length > 6) { - // If there are many options, they won't fit nicely in one line and - // should be instead listed in the description. - buffer.write("=<...>"); - } else if (option.values.isNotEmpty) { - buffer.write("=<${option.values.join('|')}>"); - } else { - buffer.write("="); - } - break; - } - - if (option.type != _OptionValueType.bool && - option.defaultValue != null && - option.defaultValue != "") { - buffer.write(" (defaults to ${option.defaultValue})"); - } - - buffer.writeln(); - buffer - .writeln(" ${option.description.replaceAll('\n', '\n ')}"); - buffer.writeln(); - } - - print(buffer); - } - - _Option _findByAbbreviation(String abbreviation) { - for (var option in _options) { - if (abbreviation == option.abbreviation) return option; - } - - return null; - } - - _Option _findByName(String name) { - for (var option in _options) { - if (name == option.name) return option; - - // Allow hyphens instead of underscores as the separator since they are - // more common for command line flags. - if (name == option.name.replaceAll("_", "-")) return option; - } - - return null; - } -} diff --git a/tools/testing/dart/package_testing_support.dart b/tools/testing/dart/package_testing_support.dart index 913d6282356..d4706635384 100644 --- a/tools/testing/dart/package_testing_support.dart +++ b/tools/testing/dart/package_testing_support.dart @@ -4,15 +4,17 @@ library package_testing_support; -import 'options.dart'; -import 'test_configurations.dart'; -import 'test_suite.dart'; +import 'test_configurations.dart' show testConfigurations; + +import 'test_options.dart' show TestOptionsParser; + +import 'test_suite.dart' show TestUtils; void main(List arguments) { TestUtils.setDartDirUri(Uri.base); var configurations = []; for (var argument in arguments) { - configurations.addAll(new OptionsParser().parse(argument.split(" "))); + configurations.addAll(new TestOptionsParser().parse(argument.split(" "))); } testConfigurations(configurations); } diff --git a/tools/testing/dart/test_options.dart b/tools/testing/dart/test_options.dart new file mode 100644 index 00000000000..06ddec629ea --- /dev/null +++ b/tools/testing/dart/test_options.dart @@ -0,0 +1,1012 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +library test_options_parser; + +import "dart:io"; +import "drt_updater.dart"; +import "test_suite.dart"; +import "path.dart"; +import "compiler_configuration.dart" show CompilerConfiguration; +import "runtime_configuration.dart" show RuntimeConfiguration; + +const List defaultTestSelectors = const [ + 'samples', + 'standalone', + 'corelib', + 'co19', + 'language', + 'isolate', + 'vm', + 'html', + 'benchmark_smoke', + 'utils', + 'lib', + 'analyze_library', + 'service', + 'kernel', + 'observatory_ui' +]; + +/** + * Specification of a single test option. + * + * The name of the specification is used as the key for the option in + * the Map returned from the [TestOptionParser] parse method. + */ +class _TestOptionSpecification { + _TestOptionSpecification( + this.name, this.description, this.keys, this.values, this.defaultValue, + {this.type: 'string'}); + String name; + String description; + List keys; + List values; + Object defaultValue; + String type; +} + +/** + * Parser of test options. + */ +class TestOptionsParser { + /** + * Creates a test options parser initialized with the known options. + */ + TestOptionsParser() { + _options = [ + new _TestOptionSpecification('mode', 'Mode in which to run the tests', + ['-m', '--mode'], ['all', 'debug', 'release', 'product'], 'debug'), + new _TestOptionSpecification( + 'compiler', + '''Specify any compilation step (if needed). + + none: Do not compile the Dart code (run native Dart code on the VM). + (only valid with the following runtimes: vm, flutter, drt) + + precompiler: Compile into AOT snapshot before running the test. + (only valid with the dart_precompiled runtime) + + dart2js: Compile dart code to JavaScript by running dart2js. + (only valid with the following runtimes: d8, drt, chrome, + safari, ie9, ie10, ie11, firefox, opera, chromeOnAndroid, + none (compile only)), + + dart2analyzer: Perform static analysis on Dart code by running the analyzer + (only valid with the following runtimes: none) + + app_jit: Compile the Dart code into an app snapshot before running test + (only valid with dart_app runtime) + + dartk: Compile the Dart source into Kernel before running test. + + dartkp: Compiler the Dart source into Kernel and then Kernel into AOT + snapshot before running the test. + (only valid with the dart_precompiled runtime) + ''', + ['-c', '--compiler'], + [ + 'none', + 'precompiler', + 'dart2js', + 'dart2analyzer', + 'app_jit', + 'dartk', + 'dartkp' + ], + 'none'), + // TODO(antonm): fix the option drt. + new _TestOptionSpecification( + 'runtime', + '''Where the tests should be run. + vm: Run Dart code on the standalone dart vm. + + flutter: Run Dart code on the flutter engine. + + dart_precompiled: Run a precompiled snapshot on a variant of the standalone + dart vm lacking a JIT. + + d8: Run JavaScript from the command line using v8. + + jsshell: Run JavaScript from the command line using firefox js-shell. + + drt: Run Dart or JavaScript in the headless version of Chrome, + Content shell. + + dartium: Run Dart or JavaScript in Dartium. + + ContentShellOnAndroid: Run Dart or JavaScript in Dartium content shell + on Android. + + DartiumOnAndroid: Run Dart or Javascript in Dartium on Android. + + [ff | chrome | safari | ie9 | ie10 | ie11 | opera | chromeOnAndroid]: + Run JavaScript in the specified browser. + + self_check: Pass each test or its compiled output to every file under + `pkg` whose name ends with `_self_check.dart`. + Each test is given to the self_check tester as a filename on stdin using + the batch-mode protocol. + + none: No runtime, compile only (for example, used for dart2analyzer static + analysis tests).''', + ['-r', '--runtime'], + [ + 'vm', + 'flutter', + 'dart_precompiled', + 'd8', + 'jsshell', + 'drt', + 'dartium', + 'ff', + 'firefox', + 'chrome', + 'safari', + 'ie9', + 'ie10', + 'ie11', + 'opera', + 'chromeOnAndroid', + 'safarimobilesim', + 'ContentShellOnAndroid', + 'DartiumOnAndroid', + 'self_check', + 'none' + ], + 'vm'), + new _TestOptionSpecification( + 'arch', + 'The architecture to run tests for', + ['-a', '--arch'], + [ + 'all', + 'ia32', + 'x64', + 'arm', + 'armv6', + 'armv5te', + 'arm64', + 'mips', + 'simarm', + 'simarmv6', + 'simarmv5te', + 'simarm64', + 'simmips', + 'simdbc', + 'simdbc64', + ], + 'x64'), + new _TestOptionSpecification( + 'system', + 'The operating system to run tests on', + ['-s', '--system'], + ['linux', 'macos', 'windows', 'android'], + Platform.operatingSystem), + new _TestOptionSpecification( + 'checked', 'Run tests in checked mode', ['--checked'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'strong', 'Run tests in strong mode', ['--strong'], [], false, + type: 'bool'), + new _TestOptionSpecification('host_checked', + 'Run compiler in checked mode', ['--host-checked'], [], false, + type: 'bool'), + new _TestOptionSpecification('minified', + 'Enable minification in the compiler', ['--minified'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'csp', + 'Run tests under Content Security Policy restrictions', + ['--csp'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'cps_ir', + 'Run the compiler with the cps based backend', + ['--cps-ir'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'fast_startup', + 'Pass the --fast-startup flag to dart2js', + ['--fast-startup'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'dart2js_with_kernel', + 'Enable the internal pipeline in dart2js to use kernel', + ['--dart2js-with-kernel'], + [], + false, + type: 'bool'), + new _TestOptionSpecification('hot_reload', 'Run hot reload stress tests', + ['--hot-reload'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'hot_reload_rollback', + 'Run hot reload rollback stress tests', + ['--hot-reload-rollback'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'use_blobs', + 'Use mmap instead of shared libraries for precompilation', + ['--use-blobs'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'timeout', 'Timeout in seconds', ['-t', '--timeout'], [], -1, + type: 'int'), + new _TestOptionSpecification( + 'progress', + 'Progress indication mode', + ['-p', '--progress'], + [ + 'compact', + 'color', + 'line', + 'verbose', + 'silent', + 'status', + 'buildbot', + 'diff' + ], + 'compact'), + new _TestOptionSpecification('failure-summary', + 'Print failure summary at the end', ['--failure-summary'], [], false, + type: 'bool'), + new _TestOptionSpecification('step_name', + 'Step name for use by -pbuildbot', ['--step_name'], [], null), + new _TestOptionSpecification( + 'report', + 'Print a summary report of the number of tests, by expectation', + ['--report'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'tasks', + 'The number of parallel tasks to run', + ['-j', '--tasks'], + [], + Platform.numberOfProcessors, + type: 'int'), + new _TestOptionSpecification( + 'shards', + 'The number of instances that the tests will be sharded over', + ['--shards'], + [], + 1, + type: 'int'), + new _TestOptionSpecification( + 'shard', + 'The index of this instance when running in sharded mode', + ['--shard'], + [], + 1, + type: 'int'), + new _TestOptionSpecification( + 'help', 'Print list of options', ['-h', '--help'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'verbose', 'Verbose output', ['-v', '--verbose'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'verify-ir', 'Verify kernel IR', ['--verify-ir'], [], false, + type: 'bool'), + new _TestOptionSpecification('no-tree-shake', + 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'list', 'List tests only, do not run them', ['--list'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'report_in_json', + 'When doing list, output result summary in json only.', + ['--report-in-json'], + [], + false, + type: 'bool'), + new _TestOptionSpecification('time', + 'Print timing information after running tests', ['--time'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'dart', 'Path to dart executable', ['--dart'], [], ''), + new _TestOptionSpecification( + 'flutter', 'Path to flutter executable', ['--flutter'], [], ''), + new _TestOptionSpecification( + 'drt', // TODO(antonm): fix the option name. + 'Path to content shell executable', + ['--drt'], + [], + ''), + new _TestOptionSpecification('dartium', + 'Path to Dartium Chrome executable', ['--dartium'], [], ''), + new _TestOptionSpecification('firefox', + 'Path to firefox browser executable', ['--firefox'], [], ''), + new _TestOptionSpecification( + 'chrome', 'Path to chrome browser executable', ['--chrome'], [], ''), + new _TestOptionSpecification( + 'safari', 'Path to safari browser executable', ['--safari'], [], ''), + new _TestOptionSpecification( + 'use_sdk', + '''Use compiler or runtime from the SDK. + +Normally, the compiler or runtimes in PRODUCT_DIR is tested, with this +option, the compiler or runtime in PRODUCT_DIR/dart-sdk/bin is tested. + +Note: currently only implemented for dart2js.''', + ['--use-sdk'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'build_directory', + 'The name of the build directory, where products are placed.', + ['--build-directory'], + [], + ''), + new _TestOptionSpecification('noBatch', 'Do not run tests in batch mode', + ['-n', '--nobatch'], [], false, + type: 'bool'), + new _TestOptionSpecification('dart2js_batch', + 'Run dart2js tests in batch mode', ['--dart2js-batch'], [], false, + type: 'bool'), + new _TestOptionSpecification( + 'append_logs', + 'Do not delete old logs but rather append to them.', + ['--append_logs'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'write_debug_log', + 'Don\'t write debug messages to stdout but rather to a logfile.', + ['--write-debug-log'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'write_test_outcome_log', + 'Write the outcome of all tests executed to a ' + '"${TestUtils.testOutcomeFileName}" file.', + ['--write-test-outcome-log'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'reset_browser_configuration', + 'Browser specific reset of configuration. ' + 'WARNING: Using this option may remove your bookmarks and ' + 'other settings.', + ['--reset-browser-configuration'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'copy_coredumps', + 'If we see a crash that we did not expect, copy the core dumps. ' + 'to /tmp', + ['--copy-coredumps'], + [], + false, + type: 'bool'), + new _TestOptionSpecification( + 'local_ip', + 'IP address the http servers should listen on.' + 'This address is also used for browsers to connect.', + ['--local_ip'], + [], + '127.0.0.1'), + new _TestOptionSpecification('test_server_port', + 'Port for test http server.', ['--test_server_port'], [], 0, + type: 'int'), + new _TestOptionSpecification( + 'test_server_cross_origin_port', + 'Port for test http server cross origin.', + ['--test_server_cross_origin_port'], + [], + 0, + type: 'int'), + new _TestOptionSpecification('test_driver_port', + 'Port for http test driver server.', ['--test_driver_port'], [], 0, + type: 'int'), + new _TestOptionSpecification( + 'test_driver_error_port', + 'Port for http test driver server errors.', + ['--test_driver_error_port'], + [], + 0, + type: 'int'), + new _TestOptionSpecification( + 'record_to_file', + 'Records all the commands that need to be executed and writes it ' + 'out to a file.', + ['--record_to_file'], + [], + null), + new _TestOptionSpecification( + 'replay_from_file', + 'Records all the commands that need to be executed and writes it ' + 'out to a file.', + ['--replay_from_file'], + [], + null), + new _TestOptionSpecification( + 'builder_tag', + 'Machine specific options that is not captured by the regular ' + 'test options. Used to be able to make sane updates to the ' + 'status files.', + ['--builder-tag'], + [], + ''), + new _TestOptionSpecification( + 'vm_options', + 'Extra options to send to the vm when running', + ['--vm-options'], + [], + null), + new _TestOptionSpecification( + 'dart2js_options', + 'Extra options for dart2js compilation step', + ['--dart2js-options'], + [], + null), + new _TestOptionSpecification( + 'suite_dir', + 'Additional directory to add to the testing matrix', + ['--suite-dir'], + [], + null), + new _TestOptionSpecification('package_root', + 'The package root to use for testing.', ['--package-root'], [], null), + new _TestOptionSpecification( + 'packages', + 'The package spec file to use for testing.', + ['--packages'], + [], + null), + new _TestOptionSpecification( + 'exclude_suite', + 'Exclude suites from default selector, only works when no' + ' selector has been specified on the command line', + ['--exclude-suite'], + [], + null), + new _TestOptionSpecification( + 'skip-compilation', + 'Skip the compilation step, using the compilation artifacts left in ' + ' the output folder from a previous run.' + 'This flag will often cause false positves and negatives, but can be' + ' useful for quick-and-dirty offline testing when not making changes' + ' that affect the compiler.', + ['--skip-compilation'], + [], + false, + type: 'bool') + ]; + } + + /** + * Parse a list of strings as test options. + * + * Returns a list of configurations in which to run the + * tests. Configurations are maps mapping from option keys to + * values. When encountering the first non-option string, the rest + * of the arguments are stored in the returned Map under the 'rest' + * key. + */ + List parse(List arguments) { + var configuration = new Map(); + // Fill in configuration with arguments passed to the test script. + var numArguments = arguments.length; + for (var i = 0; i < numArguments; i++) { + // Extract name and value for options. + String arg = arguments[i]; + String name = ''; + String value = ''; + _TestOptionSpecification spec; + if (arg.startsWith('--')) { + if (arg == '--help') { + _printHelp(); + return null; + } + var split = arg.indexOf('='); + if (split == -1) { + name = arg; + spec = _getSpecification(name); + // Boolean options do not have a value. + if (spec.type != 'bool') { + if ((i + 1) >= arguments.length) { + print('No value supplied for option $name'); + return null; + } + value = arguments[++i]; + } + } else { + name = arg.substring(0, split); + spec = _getSpecification(name); + value = arg.substring(split + 1, arg.length); + } + } else if (arg.startsWith('-')) { + if (arg == '-h') { + _printHelp(); + return null; + } + if (arg.length > 2) { + name = arg.substring(0, 2); + spec = _getSpecification(name); + value = arg.substring(2, arg.length); + } else { + name = arg; + spec = _getSpecification(name); + // Boolean options do not have a value. + if (spec.type != 'bool') { + if ((i + 1) >= arguments.length) { + print('No value supplied for option $name'); + return null; + } + value = arguments[++i]; + } + } + } else { + // The argument does not start with '-' or '--' and is + // therefore not an option. We use it as a test selection + // pattern. + var patterns = configuration.putIfAbsent('selectors', () => []); + patterns.add(arg); + continue; + } + + // Multiple uses of a flag are an error, because there is no + // naturally correct way to handle conflicting options. + if (configuration.containsKey(spec.name)) { + print('Error: test.dart disallows multiple "--${spec.name}" flags'); + exit(1); + } + // Parse the value for the option. + if (spec.type == 'bool') { + if (!value.isEmpty) { + print('No value expected for bool option $name'); + exit(1); + } + configuration[spec.name] = true; + } else if (spec.type == 'int') { + try { + configuration[spec.name] = int.parse(value); + } catch (e) { + print('Integer value expected for int option $name'); + exit(1); + } + } else { + assert(spec.type == 'string'); + if (!spec.values.isEmpty) { + for (var v in value.split(',')) { + if (spec.values.lastIndexOf(v) == -1) { + print('Unknown value ($v) for option $name'); + exit(1); + } + } + } + configuration[spec.name] = value; + } + } + + // Apply default values for unspecified options. + for (var option in _options) { + if (!configuration.containsKey(option.name)) { + configuration[option.name] = option.defaultValue; + } + } + + List expandedConfigs = _expandConfigurations(configuration); + List result = expandedConfigs.where(_isValidConfig).toList(); + for (var config in result) { + config['_reproducing_arguments_'] = + _constructReproducingCommandArguments(config); + } + return result.isEmpty ? null : result; + } + + // For printing out reproducing command lines, we don't want to add these + // options. + final _blacklistedOptions = [ + 'append_logs', + 'build_directory', + 'chrome', + 'copy_coredumps', + 'dart', + 'flutter', + 'dartium', + 'drt', + 'exclude_suite', + 'failure-summary', + 'firefox', + 'local_ip', + 'progress', + 'report', + 'safari', + 'shard', + 'shards', + 'step_name', + 'tasks', + 'time', + 'verbose', + 'write_debug_log', + 'write_test_outcome_log', + ].toSet(); + + List _constructReproducingCommandArguments(Map config) { + var arguments = new List(); + for (var option in _options) { + var name = option.name; + if (!config.containsKey(name) || _blacklistedOptions.contains(name)) { + continue; + } + var value = config[name]; + if (config[name] == option.defaultValue || + (name == 'packages' && + value == + TestUtils.dartDirUri.resolve('.packages').toFilePath())) { + continue; + } + shortest(String a, String b) => a.length <= b.length ? a : b; + var key = option.keys.reduce(shortest); + if (option.type == 'bool') { + arguments.add(key); + } else if (key.startsWith('--')) { + // long version + arguments.add(key); + arguments.add("$value"); + } else { + // short version + assert(key.startsWith('-')); + arguments.add("$key$value"); + } + } + return arguments; + } + + /** + * Determine if a particular configuration has a valid combination of compiler + * and runtime elements. + */ + bool _isValidConfig(Map config) { + bool isValid = true; + List validRuntimes; + switch (config['compiler']) { + case 'dart2js': + // Note: by adding 'none' as a configuration, if the user + // runs test.py -c dart2js -r drt,none the dart2js_none and + // dart2js_drt will be duplicating work. If later we don't need 'none' + // with dart2js, we should remove it from here. + validRuntimes = const [ + 'd8', + 'jsshell', + 'drt', + 'none', + 'dartium', + 'ff', + 'chrome', + 'safari', + 'ie9', + 'ie10', + 'ie11', + 'opera', + 'chromeOnAndroid', + 'safarimobilesim' + ]; + break; + case 'dart2analyzer': + validRuntimes = const ['none']; + break; + case 'app_jit': + case 'dartk': + validRuntimes = const ['vm', 'self_check', 'none']; + break; + case 'precompiler': + case 'dartkp': + validRuntimes = const ['dart_precompiled']; + break; + case 'none': + validRuntimes = const [ + 'vm', + 'flutter', + 'drt', + 'dartium', + 'ContentShellOnAndroid', + 'DartiumOnAndroid' + ]; + break; + } + if (!validRuntimes.contains(config['runtime'])) { + isValid = false; + print("Warning: combination of compiler '${config['compiler']}' and " + "runtime '${config['runtime']}' is invalid. " + "Skipping this combination."); + } + if (config['ie'] && Platform.operatingSystem != 'windows') { + isValid = false; + print("Warning: cannot run Internet Explorer on non-Windows operating" + " system."); + } + if (config['shard'] < 1 || config['shard'] > config['shards']) { + isValid = false; + print("Error: shard index is ${config['shard']} out of " + "${config['shards']} shards"); + } + if ((config['runtime'] == 'flutter') && (config['flutter'] == '')) { + isValid = false; + print("-rflutter requires the flutter engine executable to " + "be specified using --flutter="); + } + if ((config['runtime'] == 'flutter') && (config['arch'] != 'x64')) { + isValid = false; + print("-rflutter is applicable only for --arch=x64"); + } + + return isValid; + } + + /** + * Recursively expand a configuration with multiple values per key + * into a list of configurations with exactly one value per key. + */ + List _expandConfigurations(Map configuration) { + // Expand the pseudo-values such as 'all'. + if (configuration['arch'] == 'all') { + configuration['arch'] = 'ia32,x64,simarm,simarm64,simmips,simdbc64'; + } + if (configuration['mode'] == 'all') { + configuration['mode'] = 'debug,release,product'; + } + + if (configuration['report_in_json']) { + configuration['list'] = true; + configuration['report'] = true; + } + + // Use verbose progress indication for verbose output unless buildbot + // progress indication is requested. + if (configuration['verbose'] && configuration['progress'] != 'buildbot') { + configuration['progress'] = 'verbose'; + } + + // Create the artificial negative options that test status files + // expect. + configuration['unchecked'] = !configuration['checked']; + configuration['host_unchecked'] = !configuration['host_checked']; + configuration['unminified'] = !configuration['minified']; + configuration['nocsp'] = !configuration['csp']; + + String runtime = configuration['runtime']; + if (runtime == 'firefox') { + configuration['runtime'] == 'ff'; + } + + String compiler = configuration['compiler']; + configuration['browser'] = TestUtils.isBrowserRuntime(runtime); + configuration['analyzer'] = TestUtils.isCommandLineAnalyzer(compiler); + + // Set the javascript command line flag for less verbose status files. + configuration['jscl'] = TestUtils.isJsCommandLineRuntime(runtime); + + // Allow suppression that is valid for all ie versions + configuration['ie'] = runtime.startsWith('ie'); + + // Expand the test selectors into a suite name and a simple + // regular expressions to be used on the full path of a test file + // in that test suite. If no selectors are explicitly given use + // the default suite patterns. + var selectors = configuration['selectors']; + if (selectors is! Map) { + if (selectors == null) { + if (configuration['suite_dir'] != null) { + var suite_path = new Path(configuration['suite_dir']); + selectors = [suite_path.filename]; + } else { + selectors = defaultTestSelectors.toList(); + } + + var excludeSuites = configuration['exclude_suite'] != null + ? configuration['exclude_suite'].split(',') + : []; + for (var exclude in excludeSuites) { + if (selectors.contains(exclude)) { + selectors.remove(exclude); + } else { + print("Warning: default selectors does not contain $exclude"); + } + } + } + var selectorMap = {}; + for (var i = 0; i < selectors.length; i++) { + var pattern = selectors[i]; + var suite = pattern; + var slashLocation = pattern.indexOf('/'); + if (slashLocation != -1) { + suite = pattern.substring(0, slashLocation); + pattern = pattern.substring(slashLocation + 1); + pattern = pattern.replaceAll('*', '.*'); + } else { + pattern = ".?"; + } + if (selectorMap.containsKey(suite)) { + print("Error: '$suite/$pattern'. Only one test selection" + " pattern is allowed to start with '$suite/'"); + exit(1); + } + selectorMap[suite] = new RegExp(pattern); + } + configuration['selectors'] = selectorMap; + } + + // Put observatory_ui in a configuration with its own packages override. + // Only one value in the configuration map is mutable: + selectors = configuration['selectors']; + if (selectors.containsKey('observatory_ui')) { + if (selectors.length == 1) { + configuration['packages'] = TestUtils.dartDirUri + .resolve('runtime/observatory/.packages') + .toFilePath(); + } else { + // Make a new configuration whose selectors map only contains + // observatory_ui, and remove the key from the original selectors. + // The only mutable value in the map is the selectors, so a + // shallow copy is safe. + var observatoryConfiguration = new Map.from(configuration); + observatoryConfiguration['selectors'] = { + 'observatory_ui': selectors['observatory_ui'] + }; + selectors.remove('observatory_ui'); + + // Set the packages flag. + observatoryConfiguration['packages'] = TestUtils.dartDirUri + .resolve('runtime/observatory/.packages') + .toFilePath(); + + // Return the expansions of both configurations. Neither will reach + // this line in the recursive call to _expandConfigurations. + return _expandConfigurations(configuration) + ..addAll(_expandConfigurations(observatoryConfiguration)); + } + } + // Set the default package spec explicitly. + if (configuration['package_root'] == null && + configuration['packages'] == null) { + configuration['packages'] = + TestUtils.dartDirUri.resolve('.packages').toFilePath(); + } + + // Expand the architectures. + if (configuration['arch'].contains(',')) { + return _expandHelper('arch', configuration); + } + + // Expand modes. + if (configuration['mode'].contains(',')) { + return _expandHelper('mode', configuration); + } + + // Expand compilers. + if (configuration['compiler'].contains(',')) { + return _expandHelper('compiler', configuration); + } + + // Expand runtimes. + var runtimes = configuration['runtime']; + if (runtimes.contains(',')) { + return _expandHelper('runtime', configuration); + } else { + // All runtimes eventually go through this path, after expansion. + var updater = runtimeUpdater(configuration); + if (updater != null) { + updater.update(); + } + } + + // Adjust default timeout based on mode, compiler, and sometimes runtime. + if (configuration['timeout'] == -1) { + var isReload = + configuration['hot_reload'] || configuration['hot_reload_rollback']; + int compilerMulitiplier = + new CompilerConfiguration(configuration).computeTimeoutMultiplier(); + int runtimeMultiplier = new RuntimeConfiguration(configuration) + .computeTimeoutMultiplier( + mode: configuration['mode'], + isChecked: configuration['checked'], + isReload: isReload, + arch: configuration['arch']); + configuration['timeout'] = 60 * compilerMulitiplier * runtimeMultiplier; + } + + return [configuration]; + } + + /** + * Helper for _expandConfigurations. Creates a new configuration and adds it + * to a list, for use in a case when a particular configuration has multiple + * results (separated by a ','). + * Arguments: + * option: The particular test option we are expanding. + * configuration: The map containing all test configuration information + * specified. + */ + List _expandHelper(String option, Map configuration) { + var result = new List(); + var configs = configuration[option]; + for (var config in configs.split(',')) { + var newConfiguration = new Map.from(configuration); + newConfiguration[option] = config; + result.addAll(_expandConfigurations(newConfiguration)); + } + return result; + } + + /** + * Print out usage information. + */ + void _printHelp() { + print('usage: dart test.dart [options] [selector]'); + print(''); + print('The optional selector limits the tests that will be run.'); + print('For example, the selector "language/issue", or equivalently'); + print('"language/*issue*", limits to test files matching the regexp'); + print('".*issue.*\\.dart" in the "tests/language" directory.'); + print(''); + print('Options:\n'); + for (var option in _options) { + print('${option.name}: ${option.description}.'); + for (var name in option.keys) { + assert(name.startsWith('-')); + var buffer = new StringBuffer(); + ; + buffer.write(name); + if (option.type == 'bool') { + assert(option.values.isEmpty); + } else { + buffer.write(name.startsWith('--') ? '=' : ' '); + if (option.type == 'int') { + assert(option.values.isEmpty); + buffer.write('n (default: ${option.defaultValue})'); + } else { + buffer.write('['); + bool first = true; + for (var value in option.values) { + if (!first) buffer.write(", "); + if (value == option.defaultValue) buffer.write('*'); + buffer.write(value); + first = false; + } + buffer.write(']'); + } + } + print(buffer.toString()); + } + print(''); + } + } + + /** + * Find the test option specification for a given option key. + */ + _TestOptionSpecification _getSpecification(String name) { + for (var option in _options) { + if (option.keys.contains(name)) { + return option; + } + } + print('Unknown test option $name'); + exit(1); + return null; // Unreachable. + } + + List<_TestOptionSpecification> _options; +} diff --git a/tools/testing/dart/test_suite.dart b/tools/testing/dart/test_suite.dart index cf0af89723d..08362f36542 100644 --- a/tools/testing/dart/test_suite.dart +++ b/tools/testing/dart/test_suite.dart @@ -474,7 +474,7 @@ class CCTestSuite extends TestSuite { var args = TestUtils.standardOptions(configuration); var compilerConfiguration = new CompilerConfiguration(configuration); - if (compilerConfiguration.useDfe) { + if (compilerConfiguration.useDFE) { args.add('--use-dart-frontend'); // '--dfe' has to be the first argument for run_vm_test to pick it up. args.insert(0, '--dfe=$buildDir/gen/kernel-service.dart.snapshot'); @@ -2028,6 +2028,9 @@ class TestUtils { if (compiler == "dart2js" && configuration["csp"]) { args.add("--csp"); } + if (compiler == "dart2js" && configuration["cps_ir"]) { + args.add("--use-cps-ir"); + } if (compiler == "dart2js" && configuration["fast_startup"]) { args.add("--fast-startup"); }