From a4c7cf388f9724fa4459c212a5cb207b56b66ef3 Mon Sep 17 00:00:00 2001 From: Bob Nystrom Date: Tue, 26 Jun 2018 22:17:08 +0000 Subject: [PATCH] First stab at a parsing library for named configurations. Change-Id: I1a2660914715d529b0233645b109b13c2fa9171a Reviewed-on: https://dart-review.googlesource.com/61121 Commit-Queue: Bob Nystrom Reviewed-by: William Hesse --- .packages | 1 + pkg/pkg.status | 3 + pkg/smith/analysis_options.yaml | 3 + pkg/smith/bin/print_configurations.dart | 14 + pkg/smith/lib/configuration.dart | 795 ++++++++++++++++++++++++ pkg/smith/lib/smith.dart | 5 + pkg/smith/lib/test_matrix.dart | 55 ++ pkg/smith/pubspec.yaml | 7 + pkg/smith/test/configuration_test.dart | 369 +++++++++++ pkg/smith/test/test_matrix_test.dart | 72 +++ 10 files changed, 1324 insertions(+) create mode 100644 pkg/smith/analysis_options.yaml create mode 100644 pkg/smith/bin/print_configurations.dart create mode 100644 pkg/smith/lib/configuration.dart create mode 100644 pkg/smith/lib/smith.dart create mode 100644 pkg/smith/lib/test_matrix.dart create mode 100644 pkg/smith/pubspec.yaml create mode 100644 pkg/smith/test/configuration_test.dart create mode 100644 pkg/smith/test/test_matrix_test.dart diff --git a/.packages b/.packages index e09d3e78d6b..e5e9d80d990 100644 --- a/.packages +++ b/.packages @@ -82,6 +82,7 @@ shelf:third_party/pkg/shelf/lib shelf_packages_handler:third_party/pkg/shelf_packages_handler/lib shelf_static:third_party/pkg/shelf_static/lib shelf_web_socket:third_party/pkg/shelf_web_socket/lib +smith:pkg/smith/lib source_map_stack_trace:third_party/pkg/source_map_stack_trace/lib sourcemap_testing:pkg/sourcemap_testing/lib source_maps:third_party/pkg/source_maps/lib diff --git a/pkg/pkg.status b/pkg/pkg.status index cd5cb273136..e5a3d4fa318 100644 --- a/pkg/pkg.status +++ b/pkg/pkg.status @@ -151,6 +151,9 @@ kernel/test/*: SkipByDesign # Uses dart:io and bigints. [ $no_preview_dart_2 ] dev_compiler/*: SkipByDesign # uses Dart 2. +[ !$preview_dart_2 ] +smith/test/*: Skip # Uses optional new. + [ $arch == x64 && $runtime == vm && $system == windows && $checked ] analyzer/test/src/task/strong/inferred_type_test: Pass, Slow diff --git a/pkg/smith/analysis_options.yaml b/pkg/smith/analysis_options.yaml new file mode 100644 index 00000000000..1a46de2ee05 --- /dev/null +++ b/pkg/smith/analysis_options.yaml @@ -0,0 +1,3 @@ +analyzer: + strong-mode: + implicit-casts: false diff --git a/pkg/smith/bin/print_configurations.dart b/pkg/smith/bin/print_configurations.dart new file mode 100644 index 00000000000..38b175daf3d --- /dev/null +++ b/pkg/smith/bin/print_configurations.dart @@ -0,0 +1,14 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'package:smith/smith.dart'; + +/// A rudimentary script to print the configurations in the given test matrix +/// file. +void main(List arguments) { + // TODO(rnystrom): Validate args. Usage. + var matrix = TestMatrix.fromPath(arguments[0]); + for (var configuration in matrix.configurations) { + print(configuration); + } +} diff --git a/pkg/smith/lib/configuration.dart b/pkg/smith/lib/configuration.dart new file mode 100644 index 00000000000..0467305741d --- /dev/null +++ b/pkg/smith/lib/configuration.dart @@ -0,0 +1,795 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'dart:io'; + +// TODO(rnystrom): Differences from test.dart's version: +// - Remove special handling for "ff" as firefox. +// - "windows" -> "win". +// - "macos" -> "mac". +// - toString() on enum classes is just name. +// - builderTag defaults to empty string, not null. +// Need to migrate test.dart to not expect the above before it can use this. + +// READ ME! If you add a new field to this, make sure to add it to +// [parse()], [optionsEqual()], [hashCode], and [toString()]. A good check is to +// comment out an existing field and see what breaks. Every error is a place +// where you will need to add code for your new field. + +/// A set of options that affects how a Dart SDK test is run in a way that may +/// affect its outcome. +/// +/// This includes options like "compiler" and "runtime" which fundamentally +/// decide how a test is executed. Options are tracked because a single test +/// may have different outcomes for different configurations. For example, it +/// may currently pass on the VM but not dart2js or vice versa. +/// +/// Options that affect how a test can be run but don't affect its outcome are +/// *not* stored here. Things like how test results are displayed, where logs +/// are written, etc. live outside of this. +class Configuration { + /// Expands a configuration name "[template]" all using [optionsJson] to a + /// list of configurations. + /// + /// A template is a configuration name that contains zero or more + /// parenthesized sections. Within the parentheses are a series of options + /// separated by pipes. For example: + /// + /// strong-fasta-(linux|mac|win)-(debug|release) + /// + /// Text outside of parenthesized groups is treated literally. Each + /// parenthesized section expands to a configuration for each of the options + /// separated by pipes. If a template contains multiple parenthesized + /// sections, configurations are created for all combinations of them. The + /// above template expands to: + /// + /// strong-fasta-linux-debug + /// strong-fasta-linux-release + /// strong-fasta-mac-debug + /// strong-fasta-mac-release + /// strong-fasta-win-debug + /// strong-fasta-win-release + /// + /// After expansion, the resulting strings (and [optionsJson]) are passed to + /// [parse()] to convert each one to a full configuration. + static List expandTemplate( + String template, Map optionsJson) { + if (template.isEmpty) throw FormatException("Template must not be empty."); + + var sections = >[]; + var start = 0; + while (start < template.length) { + var openParen = template.indexOf("(", start); + + if (openParen == -1) { + // Add the last literal section. + sections.add([template.substring(start, template.length)]); + break; + } + + var closeParen = template.indexOf(")", openParen); + if (closeParen == -1) { + throw FormatException('Missing ")" in name template "$template".'); + } + + // Add the literal part before the next "(". + sections.add([template.substring(start, openParen)]); + + // Add the options within the parentheses. + sections.add(template.substring(openParen + 1, closeParen).split("|")); + + // Continue past the ")". + start = closeParen + 1; + } + + var result = []; + + // Walk through every combination of every section. + iterateSection(String prefix, int section) { + // If we pinned all the sections, parse it. + if (section >= sections.length) { + try { + result.add(Configuration.parse(prefix, optionsJson)); + } on FormatException catch (ex) { + throw FormatException( + 'Could not parse expanded configuration "$prefix" from template ' + '"$template":\n${ex.message}'); + } + return; + } + + for (var i = 0; i < sections[section].length; i++) { + iterateSection(prefix + sections[section][i], section + 1); + } + } + + iterateSection("", 0); + + return result; + } + + /// Parse a single configuration with [name] with additional options defined + /// in [optionsJson]. + /// + /// The name should be a series of words separated by hyphens. Any word that + /// matches the name of an [Architecture], [Compiler], [Mode], [Runtime], or + /// [System] sets that option in the resulting configuration. Those options + /// may also be specified in the JSON map. + /// + /// Additional Boolean and string options are defined in the map. The key + /// names match the corresponding command-line option names, using kebab-case. + static Configuration parse(String name, Map optionsJson) { + if (name.isEmpty) throw FormatException("Name must not be empty."); + + // Infer option values from the words in the configuration name. + var words = name.split("-").toSet(); + var optionsCopy = new Map.of(optionsJson); + + T enumOption( + String option, List allowed, T Function(String) parse) { + // Look up the value from the words in the name. + T fromName; + for (var value in allowed) { + // Don't treat "none" as matchable since it's ambiguous as to whether + // it refers to compiler or runtime. + if (value == "none") continue; + + if (words.contains(value)) { + if (fromName != null) { + throw FormatException( + 'Found multiple values for $option ("$fromName" and "$value"), ' + 'in configuration name.'); + } + fromName = parse(value); + } + } + + // Look up the value from the options. + T fromOption; + if (optionsCopy.containsKey(option)) { + fromOption = parse(optionsCopy[option] as String); + optionsCopy.remove(option); + } + + if (fromName != null && fromOption != null) { + if (fromName == fromOption) { + throw FormatException( + 'Redundant $option in configuration name "$fromName" and options.'); + } else { + throw FormatException( + 'Found $option "$fromOption" in options and "$fromName" in ' + 'configuration name.'); + } + } + + return fromName ?? fromOption; + } + + bool boolOption(String option) { + if (!optionsCopy.containsKey(option)) return null; + + var value = optionsCopy.remove(option); + if (value == null) throw FormatException('Option "$option" was null.'); + if (value is! bool) { + throw FormatException('Option "$option" had value "$value", which is ' + 'not a bool.'); + } + return value as bool; + } + + int intOption(String option) { + if (!optionsCopy.containsKey(option)) return null; + + var value = optionsCopy.remove(option); + if (value == null) throw FormatException('Option "$option" was null.'); + if (value is! int) { + throw FormatException('Option "$option" had value "$value", which is ' + 'not an int.'); + } + return value as int; + } + + String stringOption(String option) { + if (!optionsCopy.containsKey(option)) return null; + + var value = optionsCopy.remove(option); + if (value == null) throw FormatException('Option "$option" was null.'); + if (value is! String) { + throw FormatException('Option "$option" had value "$value", which is ' + 'not a string.'); + } + return value as String; + } + + // Extract options from the name and map. + var architecture = + enumOption("architecture", Architecture.names, Architecture.find); + var compiler = enumOption("compiler", Compiler.names, Compiler.find); + var mode = enumOption("mode", Mode.names, Mode.find); + var runtime = enumOption("runtime", Runtime.names, Runtime.find); + var system = enumOption("system", System.names, System.find); + + // Fill in any missing values using defaults when possible. + architecture ??= Architecture.x64; + system ??= System.host; + + // Infer from compiler from runtime or vice versa. + if (compiler == null) { + if (runtime == null) { + throw FormatException( + 'Must specify at least one of compiler or runtime in options or ' + 'configuration name.'); + } else { + compiler = runtime.defaultCompiler; + } + } else { + if (runtime == null) { + runtime = compiler.defaultRuntime; + } else { + // Do nothing, specified both. + } + } + + // Infer the mode from the compiler. + mode ??= compiler.defaultMode; + + var configuration = Configuration( + name, architecture, compiler, mode, runtime, system, + builderTag: stringOption("builder-tag"), + vmOptions: stringOption("vm-options"), + timeout: intOption("timeout"), + enableAsserts: boolOption("enable-asserts"), + isChecked: boolOption("checked"), + isCsp: boolOption("csp"), + isHostChecked: boolOption("host-checked"), + isMinified: boolOption("minified"), + isStrong: boolOption("strong"), + previewDart2: boolOption("preview-dart-2"), + useBlobs: boolOption("use-blobs"), + useDart2JSWithKernel: boolOption("dart2js-with-kernel"), + useDart2JSOldFrontEnd: boolOption("dart2js-old-frontend"), + useFastStartup: boolOption("fast-startup"), + useHotReload: boolOption("hot-reload"), + useHotReloadRollback: boolOption("hot-reload-rollback"), + useSdk: boolOption("use-sdk")); + + // Should have consumed the whole map. + if (optionsCopy.isNotEmpty) { + throw new FormatException('Unknown option "${optionsCopy.keys.first}".'); + } + + return configuration; + } + + final String name; + + final Architecture architecture; + + final Compiler compiler; + + final Mode mode; + + final Runtime runtime; + + final System system; + + // TODO(rnystrom): Is this still needed? + final String builderTag; + + final String vmOptions; + + final int timeout; + + final bool enableAsserts; + + // TODO(rnystrom): Remove this when Dart 1.0 is no longer supported. + final bool isChecked; + + final bool isCsp; + + // TODO(rnystrom): Remove this when Dart 1.0 is no longer supported. + final bool isHostChecked; + + final bool isMinified; + + // TODO(rnystrom): Remove this when Dart 1.0 is no longer supported. + final bool isStrong; + + // TODO(rnystrom): Remove this when Dart 1.0 is no longer supported. + final bool previewDart2; + + // TODO(rnystrom): What is this? + final bool useBlobs; + + // TODO(rnystrom): Remove these when Dart 1.0 is no longer supported. + final bool useDart2JSWithKernel; + final bool useDart2JSOldFrontEnd; + + final bool useFastStartup; + + final bool useHotReload; + final bool useHotReloadRollback; + + final bool useSdk; + + Configuration(this.name, this.architecture, this.compiler, this.mode, + this.runtime, this.system, + {String builderTag, + String vmOptions, + int timeout, + bool enableAsserts, + bool isChecked, + bool isCsp, + bool isHostChecked, + bool isMinified, + bool isStrong, + bool previewDart2, + bool useBlobs, + bool useDart2JSWithKernel, + bool useDart2JSOldFrontEnd, + bool useFastStartup, + bool useHotReload, + bool useHotReloadRollback, + bool useSdk}) + : builderTag = builderTag ?? "", + vmOptions = vmOptions ?? "", + timeout = timeout ?? 0, + enableAsserts = enableAsserts ?? false, + isChecked = isChecked ?? false, + isCsp = isCsp ?? false, + isHostChecked = isHostChecked ?? false, + isMinified = isMinified ?? false, + isStrong = isStrong ?? false, + previewDart2 = previewDart2 ?? false, + useBlobs = useBlobs ?? false, + useDart2JSWithKernel = useDart2JSWithKernel ?? false, + useDart2JSOldFrontEnd = useDart2JSOldFrontEnd ?? false, + useFastStartup = useFastStartup ?? false, + useHotReload = useHotReload ?? false, + useHotReloadRollback = useHotReloadRollback ?? false, + useSdk = useSdk ?? false; + + /// Returns `true` if this configuration's options all have the same values + /// as [other]. + bool optionsEqual(Configuration other) => + architecture == other.architecture && + compiler == other.compiler && + mode == other.mode && + runtime == other.runtime && + system == other.system && + builderTag == other.builderTag && + vmOptions == other.vmOptions && + timeout == other.timeout && + enableAsserts == other.enableAsserts && + isChecked == other.isChecked && + isCsp == other.isCsp && + isHostChecked == other.isHostChecked && + isMinified == other.isMinified && + isStrong == other.isStrong && + previewDart2 == other.previewDart2 && + useBlobs == other.useBlobs && + useDart2JSWithKernel == other.useDart2JSWithKernel && + useDart2JSOldFrontEnd == other.useDart2JSOldFrontEnd && + useFastStartup == other.useFastStartup && + useHotReload == other.useHotReload && + useHotReloadRollback == other.useHotReloadRollback && + useSdk == other.useSdk; + + bool operator ==(Object other) => + other is Configuration && name == other.name && optionsEqual(other); + + int get hashCode => + name.hashCode ^ + architecture.hashCode ^ + compiler.hashCode ^ + mode.hashCode ^ + runtime.hashCode ^ + system.hashCode ^ + builderTag.hashCode ^ + vmOptions.hashCode ^ + timeout.hashCode ^ + (enableAsserts ? 1 : 0) ^ + (isChecked ? 2 : 0) ^ + (isCsp ? 4 : 0) ^ + (isHostChecked ? 8 : 0) ^ + (isMinified ? 16 : 0) ^ + (isStrong ? 32 : 0) ^ + (previewDart2 ? 64 : 0) ^ + (useBlobs ? 128 : 0) ^ + (useDart2JSWithKernel ? 256 : 0) ^ + (useDart2JSOldFrontEnd ? 512 : 0) ^ + (useFastStartup ? 1024 : 0) ^ + (useHotReload ? 2048 : 0) ^ + (useHotReloadRollback ? 4096 : 0) ^ + (useSdk ? 8192 : 0); + + String toString() { + var buffer = new StringBuffer(); + buffer.write(name); + buffer.write("("); + + var fields = []; + fields.add("architecture: $architecture"); + fields.add("compiler: $compiler"); + fields.add("mode: $mode"); + fields.add("runtime: $runtime"); + fields.add("system: $system"); + + if (builderTag != "") fields.add("builder-tag: $builderTag"); + if (vmOptions != "") fields.add("vm-options: $vmOptions"); + if (timeout != 0) fields.add("timeout: $timeout"); + if (enableAsserts) fields.add("enable-asserts"); + if (isChecked) fields.add("checked"); + if (isCsp) fields.add("csp"); + if (isHostChecked) fields.add("host-checked"); + if (isMinified) fields.add("minified"); + if (isStrong) fields.add("strong"); + if (previewDart2) fields.add("preview-dart-2"); + if (useBlobs) fields.add("use-blobs"); + if (useDart2JSWithKernel) fields.add("dart2js-with-kernel"); + if (useDart2JSOldFrontEnd) fields.add("dart2js-old-frontend"); + if (useFastStartup) fields.add("fast-startup"); + if (useHotReload) fields.add("hot-reload"); + if (useHotReloadRollback) fields.add("hot-reload-rollback"); + if (useSdk) fields.add("use-sdk"); + + buffer.write(fields.join(", ")); + buffer.write(")"); + return buffer.toString(); + } +} + +class Architecture extends NamedEnum { + static const ia32 = const Architecture._('ia32'); + static const x64 = const Architecture._('x64'); + static const arm = const Architecture._('arm'); + static const armv6 = const Architecture._('armv6'); + static const armv5te = const Architecture._('armv5te'); + static const arm64 = const Architecture._('arm64'); + static const simarm = const Architecture._('simarm'); + static const simarmv6 = const Architecture._('simarmv6'); + static const simarmv5te = const Architecture._('simarmv5te'); + static const simarm64 = const Architecture._('simarm64'); + static const simdbc = const Architecture._('simdbc'); + static const simdbc64 = const Architecture._('simdbc64'); + + static final List names = _all.keys.toList(); + + static final _all = new Map.fromIterable([ + ia32, + x64, + arm, + armv6, + armv5te, + arm64, + simarm, + simarmv6, + simarmv5te, + simarm64, + simdbc, + simdbc64 + ], key: (architecture) => (architecture as Architecture).name); + + static Architecture find(String name) { + var architecture = _all[name]; + if (architecture != null) return architecture; + + throw new ArgumentError('Unknown architecture "$name".'); + } + + const Architecture._(String name) : super(name); +} + +class Compiler extends NamedEnum { + static const none = const Compiler._('none'); + static const precompiler = const Compiler._('precompiler'); + static const dart2js = const Compiler._('dart2js'); + static const dart2analyzer = const Compiler._('dart2analyzer'); + static const dartdevc = const Compiler._('dartdevc'); + static const dartdevk = const Compiler._('dartdevk'); + static const appJit = const Compiler._('app_jit'); + static const appJitk = const Compiler._('app_jitk'); + static const dartk = const Compiler._('dartk'); + static const dartkp = const Compiler._('dartkp'); + static const specParser = const Compiler._('spec_parser'); + static const fasta = const Compiler._('fasta'); + + static final List names = _all.keys.toList(); + + static final _all = new Map.fromIterable([ + none, + precompiler, + dart2js, + dart2analyzer, + dartdevc, + dartdevk, + appJit, + appJitk, + dartk, + dartkp, + specParser, + fasta, + ], key: (compiler) => (compiler as Compiler).name); + + static Compiler find(String name) { + var compiler = _all[name]; + if (compiler != null) return compiler; + + throw new ArgumentError('Unknown compiler "$name".'); + } + + const Compiler._(String name) : super(name); + + /// Gets the runtimes this compiler can target. + List get supportedRuntimes { + switch (this) { + case Compiler.dart2js: + // Note: by adding 'none' as a configuration, if the user + // runs test.py -c dart2js -r drt,none the dart2js_none and + // dart2js_drt will be duplicating work. If later we don't need 'none' + // with dart2js, we should remove it from here. + return const [ + Runtime.d8, + Runtime.jsshell, + Runtime.none, + Runtime.firefox, + Runtime.chrome, + Runtime.safari, + Runtime.ie9, + Runtime.ie10, + Runtime.ie11, + Runtime.chromeOnAndroid, + ]; + + case Compiler.dartdevc: + case Compiler.dartdevk: + // TODO(rnystrom): Expand to support other JS execution environments + // (other browsers, d8) when tested and working. + return const [ + Runtime.none, + Runtime.chrome, + ]; + + case Compiler.dart2analyzer: + return const [Runtime.none]; + case Compiler.appJit: + case Compiler.appJitk: + case Compiler.dartk: + return const [Runtime.vm, Runtime.selfCheck]; + case Compiler.precompiler: + case Compiler.dartkp: + return const [Runtime.dartPrecompiled]; + case Compiler.specParser: + return const [Runtime.none]; + case Compiler.fasta: + return const [Runtime.none]; + case Compiler.none: + return const [Runtime.vm, Runtime.flutter]; + } + + throw "unreachable"; + } + + /// The preferred runtime to use with this compiler if no other runtime is + /// specified. + Runtime get defaultRuntime { + switch (this) { + case Compiler.dart2js: + return Runtime.d8; + case Compiler.dartdevc: + case Compiler.dartdevk: + return Runtime.chrome; + case Compiler.dart2analyzer: + return Runtime.none; + case Compiler.appJit: + case Compiler.appJitk: + case Compiler.dartk: + return Runtime.vm; + case Compiler.precompiler: + case Compiler.dartkp: + return Runtime.dartPrecompiled; + case Compiler.specParser: + case Compiler.fasta: + return Runtime.none; + case Compiler.none: + return Runtime.vm; + } + + throw "unreachable"; + } + + Mode get defaultMode { + switch (this) { + case Compiler.dart2analyzer: + case Compiler.dart2js: + case Compiler.dartdevc: + case Compiler.dartdevk: + case Compiler.fasta: + return Mode.release; + + default: + return Mode.debug; + } + } +} + +class Mode extends NamedEnum { + static const debug = const Mode._('debug'); + static const product = const Mode._('product'); + static const release = const Mode._('release'); + + static final List names = _all.keys.toList(); + + static final _all = new Map.fromIterable( + [debug, product, release], + key: (mode) => (mode as Mode).name); + + static Mode find(String name) { + var mode = _all[name]; + if (mode != null) return mode; + + throw new ArgumentError('Unknown mode "$name".'); + } + + const Mode._(String name) : super(name); + + bool get isDebug => this == debug; +} + +class Runtime extends NamedEnum { + static const vm = const Runtime._('vm'); + static const flutter = const Runtime._('flutter'); + static const dartPrecompiled = const Runtime._('dart_precompiled'); + static const d8 = const Runtime._('d8'); + static const jsshell = const Runtime._('jsshell'); + static const firefox = const Runtime._('firefox'); + static const chrome = const Runtime._('chrome'); + static const safari = const Runtime._('safari'); + static const ie9 = const Runtime._('ie9'); + static const ie10 = const Runtime._('ie10'); + static const ie11 = const Runtime._('ie11'); + static const chromeOnAndroid = const Runtime._('chromeOnAndroid'); + static const selfCheck = const Runtime._('self_check'); + static const none = const Runtime._('none'); + + static final List names = _all.keys.toList(); + + static final _all = new Map.fromIterable([ + vm, + flutter, + dartPrecompiled, + d8, + jsshell, + firefox, + chrome, + safari, + ie9, + ie10, + ie11, + chromeOnAndroid, + selfCheck, + none + ], key: (runtime) => (runtime as Runtime).name); + + static Runtime find(String name) { + var runtime = _all[name]; + if (runtime != null) return runtime; + + throw new ArgumentError('Unknown runtime "$name".'); + } + + const Runtime._(String name) : super(name); + + bool get isBrowser => const [ + ie9, + ie10, + ie11, + safari, + chrome, + firefox, + chromeOnAndroid + ].contains(this); + + bool get isIE => name.startsWith("ie"); + + bool get isSafari => name.startsWith("safari"); + + /// Whether this runtime is a command-line JavaScript environment. + bool get isJSCommandLine => const [d8, jsshell].contains(this); + + /// If the runtime doesn't support `Window.open`, we use iframes instead. + bool get requiresIFrame => !const [ie11, ie10].contains(this); + + /// The preferred compiler to use with this runtime if no other compiler is + /// specified. + Compiler get defaultCompiler { + switch (this) { + case vm: + case flutter: + return Compiler.none; + + case dartPrecompiled: + return Compiler.precompiler; + + case d8: + case jsshell: + case firefox: + case chrome: + case safari: + case ie9: + case ie10: + case ie11: + case chromeOnAndroid: + return Compiler.dart2js; + + case selfCheck: + return Compiler.dartk; + + case none: + // If we aren't running it, we probably just want to analyze it. + return Compiler.dart2analyzer; + } + + throw "unreachable"; + } +} + +class System extends NamedEnum { + static const android = const System._('android'); + static const fuchsia = const System._('fuchsia'); + static const linux = const System._('linux'); + static const mac = const System._('mac'); + static const win = const System._('win'); + + static final List names = _all.keys.toList(); + + static final _all = new Map.fromIterable( + [android, fuchsia, linux, mac, win], + key: (system) => (system as System).name); + + /// Gets the system of the current machine. + static System get host => find(Platform.operatingSystem); + + static System find(String name) { + var system = _all[name]; + if (system != null) return system; + + // Also allow dart:io's names for the operating systems. + switch (Platform.operatingSystem) { + case "macos": + return mac; + case "windows": + return win; + } + // TODO(rnystrom): What about ios? + + throw new ArgumentError('Unknown operating system "$name".'); + } + + const System._(String name) : super(name); + + /// The root directory name for build outputs on this system. + String get outputDirectory { + switch (this) { + case android: + case fuchsia: + case linux: + case win: + return 'out/'; + + case mac: + return 'xcodebuild/'; + } + + throw "unreachable"; + } +} + +/// Base class for an enum-like class whose values are identified by name. +abstract class NamedEnum { + final String name; + + const NamedEnum(this.name); + + String toString() => name; +} diff --git a/pkg/smith/lib/smith.dart b/pkg/smith/lib/smith.dart new file mode 100644 index 00000000000..7118870b3ad --- /dev/null +++ b/pkg/smith/lib/smith.dart @@ -0,0 +1,5 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +export 'configuration.dart'; +export 'test_matrix.dart'; diff --git a/pkg/smith/lib/test_matrix.dart b/pkg/smith/lib/test_matrix.dart new file mode 100644 index 00000000000..0bf2a5bd90e --- /dev/null +++ b/pkg/smith/lib/test_matrix.dart @@ -0,0 +1,55 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'dart:convert'; +import 'dart:io'; + +import 'configuration.dart'; + +/// The manifest that defines the set of supported test [Configuration]s and +/// how they are run on the bots. +class TestMatrix { + final List configurations; + + /// Reads a test matrix from the file at [path]. + static TestMatrix fromPath(String path) { + var json = jsonDecode(new File(path).readAsStringSync()); + return fromJson(json as Map); + } + + static TestMatrix fromJson(Map json) { + var configurationsJson = json["configurations"] as Map; + + // Keep track of the configurations and which templates they were expanded + // from. + var configurations = []; + + configurationsJson.forEach((template, configurationJson) { + var options = configurationJson["options"] ?? const {}; + + for (var configuration in Configuration.expandTemplate( + template, options as Map)) { + for (var existing in configurations) { + // Make the names don't collide. + if (configuration.name == existing.name) { + throw FormatException( + 'Configuration "${configuration.name}" already exists.'); + } + + // Make sure we don't have two equivalent configurations. + if (configuration.optionsEqual(existing)) { + throw FormatException( + 'Configuration "${configuration.name}" is identical to ' + '"${existing.name}".'); + } + } + + configurations.add(configuration); + } + }); + + return TestMatrix._(configurations); + } + + TestMatrix._(this.configurations); +} diff --git a/pkg/smith/pubspec.yaml b/pkg/smith/pubspec.yaml new file mode 100644 index 00000000000..42a702f9b2d --- /dev/null +++ b/pkg/smith/pubspec.yaml @@ -0,0 +1,7 @@ +name: smith +author: Dart Team +description: Shared code for working with the Dart SDK's tests and test runner. +homepage: http://www.dartlang.org +dev_dependencies: + expect: + path: ../expect diff --git a/pkg/smith/test/configuration_test.dart b/pkg/smith/test/configuration_test.dart new file mode 100644 index 00000000000..56404d00008 --- /dev/null +++ b/pkg/smith/test/configuration_test.dart @@ -0,0 +1,369 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'package:expect/minitest.dart'; + +import 'package:smith/smith.dart'; + +void main() { + group("Configuration", () { + test("equality", () { + // Same. + expect( + Configuration("name", Architecture.x64, Compiler.dart2js, Mode.debug, + Runtime.vm, System.linux), + equals(Configuration("name", Architecture.x64, Compiler.dart2js, + Mode.debug, Runtime.vm, System.linux))); + + // Mode debug != release. + expect( + Configuration("name", Architecture.x64, Compiler.dart2js, Mode.debug, + Runtime.vm, System.linux), + notEquals(Configuration("name", Architecture.x64, Compiler.dart2js, + Mode.release, Runtime.vm, System.linux))); + + // Differ by non-required option. + expect( + Configuration("name", Architecture.x64, Compiler.dart2js, Mode.debug, + Runtime.vm, System.linux, enableAsserts: true), + notEquals(Configuration("name", Architecture.x64, Compiler.dart2js, + Mode.debug, Runtime.vm, System.linux, + enableAsserts: false))); + }); + + group(".expandTemplate()", () { + test("empty string", () { + expectExpandError("", {}, 'Template must not be empty.'); + }); + + test("missing ')'", () { + expectExpandError( + "before-(oops", {}, 'Missing ")" in name template "before-(oops".'); + }); + + test("no parentheses", () { + expect( + Configuration.expandTemplate("x64-dart2js-debug-vm-linux", {}), + equals([ + Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux) + ])); + }); + + test("parentheses at beginning", () { + expect( + Configuration.expandTemplate( + "(ia32|x64)-dart2js-debug-vm-linux", {}), + equals([ + Configuration("ia32-dart2js-debug-vm-linux", Architecture.ia32, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux), + Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux) + ])); + }); + + test("parentheses at end", () { + expect( + Configuration.expandTemplate( + "x64-dart2js-debug-vm-(linux|mac|win)", {}), + equals([ + Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux), + Configuration("x64-dart2js-debug-vm-mac", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.mac), + Configuration("x64-dart2js-debug-vm-win", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.win) + ])); + }); + + test("expands all parenthesized sections", () { + expect( + Configuration.expandTemplate( + "(ia32|x64)-dart2js-(debug|release)-vm-(linux|mac|win)", {}), + equals([ + Configuration("ia32-dart2js-debug-vm-linux", Architecture.ia32, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux), + Configuration("ia32-dart2js-debug-vm-mac", Architecture.ia32, + Compiler.dart2js, Mode.debug, Runtime.vm, System.mac), + Configuration("ia32-dart2js-debug-vm-win", Architecture.ia32, + Compiler.dart2js, Mode.debug, Runtime.vm, System.win), + Configuration("ia32-dart2js-release-vm-linux", Architecture.ia32, + Compiler.dart2js, Mode.release, Runtime.vm, System.linux), + Configuration("ia32-dart2js-release-vm-mac", Architecture.ia32, + Compiler.dart2js, Mode.release, Runtime.vm, System.mac), + Configuration("ia32-dart2js-release-vm-win", Architecture.ia32, + Compiler.dart2js, Mode.release, Runtime.vm, System.win), + Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux), + Configuration("x64-dart2js-debug-vm-mac", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.mac), + Configuration("x64-dart2js-debug-vm-win", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.win), + Configuration("x64-dart2js-release-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.release, Runtime.vm, System.linux), + Configuration("x64-dart2js-release-vm-mac", Architecture.x64, + Compiler.dart2js, Mode.release, Runtime.vm, System.mac), + Configuration("x64-dart2js-release-vm-win", Architecture.x64, + Compiler.dart2js, Mode.release, Runtime.vm, System.win) + ])); + }); + test("empty '()' is treated as empty string", () { + expect( + Configuration.expandTemplate("x64-()dart2js-debug-vm-linux", {}), + equals([ + Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux) + ])); + }); + }); + + group(".parse()", () { + test("infer required fields from name", () { + expect( + Configuration.parse("ia32-dart2js-debug-vm-linux", {}), + equals(Configuration( + "ia32-dart2js-debug-vm-linux", + Architecture.ia32, + Compiler.dart2js, + Mode.debug, + Runtime.vm, + System.linux))); + }); + + test("read required fields from options", () { + expect( + Configuration.parse("something", { + "architecture": "x64", + "compiler": "dart2js", + "mode": "debug", + "runtime": "vm", + "system": "linux" + }), + equals(Configuration("something", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux))); + }); + + test("required fields from both name and options", () { + expect( + Configuration.parse("dart2js-vm", + {"architecture": "x64", "mode": "debug", "system": "linux"}), + equals(Configuration("dart2js-vm", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux))); + }); + + test("'none' is not treated as compiler or runtime name", () { + expect( + Configuration.parse("none-x64-dart2js-debug-vm-linux", {}), + equals(Configuration( + "none-x64-dart2js-debug-vm-linux", + Architecture.x64, + Compiler.dart2js, + Mode.debug, + Runtime.vm, + System.linux))); + }); + + test("architecture defaults to 'x64'", () { + expect(Configuration.parse("dart2js-debug-vm-linux", {}).architecture, + equals(Architecture.x64)); + }); + + test("compiler defaults to runtime's default compiler", () { + expect(Configuration.parse("vm", {}).compiler, equals(Compiler.none)); + }); + + test("mode defaults to compiler's default mode", () { + expect(Configuration.parse("dartkp-vm-linux", {}).mode, + equals(Mode.debug)); + + expect(Configuration.parse("dart2js-vm-linux", {}).mode, + equals(Mode.release)); + }); + + test("runtime defaults to compiler's default runtime", () { + expect(Configuration.parse("dartdevc", {}).runtime, + equals(Runtime.chrome)); + }); + + test("runtime defaults to compiler's default runtime from option", () { + expect(Configuration.parse("wat", {"compiler": "dartdevc"}).runtime, + equals(Runtime.chrome)); + }); + + test("system defaults to the host os", () { + expect( + Configuration.parse("dart2js-vm", {}).system, equals(System.host)); + }); + + test("other options from map", () { + expect( + Configuration.parse("dart2js", { + "builder-tag": "the tag", + "vm-options": "vm stuff", + "enable-asserts": true, + "checked": true, + "csp": true, + "host-checked": true, + "minified": true, + "preview-dart-2": true, + "dart2js-with-kernel": true, + "dart2js-old-frontend": true, + "fast-startup": true, + "hot-reload": true, + "hot-reload-rollback": true, + "use-sdk": true, + }), + equals(Configuration( + "dart2js", + Architecture.x64, + Compiler.dart2js, + Mode.release, + Runtime.d8, + System.host, + builderTag: "the tag", + vmOptions: "vm stuff", + enableAsserts: true, + isChecked: true, + isCsp: true, + isHostChecked: true, + isMinified: true, + previewDart2: true, + useDart2JSWithKernel: true, + useDart2JSOldFrontEnd: true, + useFastStartup: true, + useHotReload: true, + useHotReloadRollback: true, + useSdk: true, + ))); + }); + + test("neither compiler nor runtime specified", () { + expectParseError( + "debug", + {}, + 'Must specify at least one of compiler or runtime in options or ' + 'configuration name.'); + }); + + test("empty string", () { + expectParseError("", {}, 'Name must not be empty.'); + }); + + test("redundant field", () { + expectParseError("dart2js-debug", {"mode": "debug"}, + 'Redundant mode in configuration name "debug" and options.'); + }); + + test("duplicate field", () { + expectParseError( + "dart2js-debug", + {"mode": "release"}, + 'Found mode "release" in options and "debug" in configuration ' + 'name.'); + }); + + test("multiple values for same option in name", () { + expectParseError( + "dart2js-debug-release", + {}, + 'Found multiple values for mode ("debug" and "release"), in ' + 'configuration name.'); + }); + + test("null bool option", () { + expectParseError("dart2js", {"enable-asserts": null}, + 'Option "enable-asserts" was null.'); + }); + + test("wrong type for bool option", () { + expectParseError("dart2js", {"enable-asserts": "false"}, + 'Option "enable-asserts" had value "false", which is not a bool.'); + }); + + test("null string option", () { + expectParseError( + "dart2js", {"builder-tag": null}, 'Option "builder-tag" was null.'); + }); + + test("wrong type for string option", () { + expectParseError("dart2js", {"builder-tag": true}, + 'Option "builder-tag" had value "true", which is not a string.'); + }); + + test("unknown option", () { + expectParseError("dart2js", {"wat": "???"}, 'Unknown option "wat".'); + }); + }); + + group("constructor", () {}); + + group("optionsEqual()", () { + var debugWithAsserts = Configuration( + "name", + Architecture.x64, + Compiler.dart2js, + Mode.debug, + Runtime.vm, + System.linux, + enableAsserts: true, + ); + + var debugWithAsserts2 = Configuration( + "different name", + Architecture.x64, + Compiler.dart2js, + Mode.debug, + Runtime.vm, + System.linux, + enableAsserts: true, + ); + + var debugNoAsserts = Configuration( + "name", + Architecture.x64, + Compiler.dart2js, + Mode.debug, + Runtime.vm, + System.linux, + ); + + var releaseNoAsserts = Configuration( + "name", + Architecture.x64, + Compiler.dart2js, + Mode.release, + Runtime.vm, + System.linux, + ); + + test("different options are not equal", () { + expect(debugWithAsserts.optionsEqual(debugNoAsserts), isFalse); + expect(debugNoAsserts.optionsEqual(releaseNoAsserts), isFalse); + expect(releaseNoAsserts.optionsEqual(debugWithAsserts), isFalse); + }); + + test("same options are equal", () { + expect(debugWithAsserts.optionsEqual(debugWithAsserts2), isTrue); + }); + }); + }); +} + +void expectParseError(String name, Map options, String error) { + try { + var configuration = Configuration.parse(name, options); + fail("Expected FormatException but got $configuration."); + } on FormatException catch (ex) { + expect(ex.message, equals(error)); + } +} + +void expectExpandError( + String template, Map options, String error) { + try { + var configurations = Configuration.expandTemplate(template, options); + fail("Expected FormatException but got $configurations."); + } on FormatException catch (ex) { + expect(ex.message, equals(error)); + } +} diff --git a/pkg/smith/test/test_matrix_test.dart b/pkg/smith/test/test_matrix_test.dart new file mode 100644 index 00000000000..3872422ae38 --- /dev/null +++ b/pkg/smith/test/test_matrix_test.dart @@ -0,0 +1,72 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. +import 'package:expect/minitest.dart'; + +import 'package:smith/smith.dart'; + +void main() { + group("TestMatrix.fromJson()", () { + test("parses configurations", () { + var testMatrix = TestMatrix.fromJson({ + "configurations": { + "x64-dart2js-debug-vm-linux": { + "options": {"enableAsserts": true}, + }, + "x64-dartdevc-vm-linux": { + "options": { + "mode": "release", + "enableAsserts": true + }, + }, + } + }); + + expect( + testMatrix.configurations[0], + equals(Configuration("x64-dart2js-debug-vm-linux", Architecture.x64, + Compiler.dart2js, Mode.debug, Runtime.vm, System.linux, + enableAsserts: true))); + expect( + testMatrix.configurations[1], + equals(Configuration("x64-dartdevc-vm-linux", Architecture.x64, + Compiler.dartdevc, Mode.release, Runtime.vm, System.linux, + enableAsserts: true))); + }); + + test("error if expanded configuration names collide", () { + expectJsonError( + 'Configuration "none-x64-dart2js-debug-vm-linux" already exists.', { + "configurations": { + "none-x64-dart2js-debug-vm-linux": {}, + "none-(x64|ia32)-dart2js-debug-vm-linux": {}, + } + }); + }); + + test("error if two configurations have same options", () { + expectJsonError( + 'Configuration "two-x64-dart2js-debug-vm-linux" is identical to ' + '"one-x64-dart2js-debug-vm-linux".', + { + "configurations": { + "one-x64-dart2js-debug-vm-linux": { + "options": {"enableAsserts": true} + }, + "two-x64-dart2js-debug-vm-linux": { + "options": {"enableAsserts": true} + }, + } + }); + }); + }); +} + +void expectJsonError(String error, Map json) { + try { + var testMatrix = TestMatrix.fromJson(json); + fail("Expected FormatException but got $testMatrix."); + } on FormatException catch (ex) { + expect(ex.message, equals(error)); + } +}