diff --git a/.packages b/.packages index 37c475e3971..80bae2dd22b 100644 --- a/.packages +++ b/.packages @@ -94,10 +94,10 @@ term_glyph:third_party/pkg/term_glyph/lib test:third_party/pkg/test/pkgs/test/lib test_api:third_party/pkg/test/pkgs/test_api/lib test_core:third_party/pkg/test/pkgs/test_core/lib -test_dart:tools/testing/dart test_descriptor:third_party/pkg/test_descriptor/lib test_process:third_party/pkg/test_process/lib test_reflective_loader:third_party/pkg/test_reflective_loader/lib +test_runner:pkg/test_runner/lib testing:pkg/testing/lib typed_data:third_party/pkg/typed_data/lib unittest:third_party/pkg/unittest/lib diff --git a/pkg/dev_compiler/test/sourcemap/ddc_common.dart b/pkg/dev_compiler/test/sourcemap/ddc_common.dart index 4308f5b5d2f..2609d4e4747 100644 --- a/pkg/dev_compiler/test/sourcemap/ddc_common.dart +++ b/pkg/dev_compiler/test/sourcemap/ddc_common.dart @@ -163,7 +163,7 @@ String getWrapperContent( void createHtmlWrapper(File sdkJsFile, Uri outputFile, String jsContent, String outputFilename, Uri outDir) { - // For debugging via HTML, Chrome and ./tools/testing/dart/http_server.dart. + // For debugging via HTML, Chrome and ./pkg/test_runner/bin/http_server.dart. var sdkFile = File(path.relative(sdkJsFile.path, from: sdkRoot.path)); String jsRootDart = "/root_dart/${sdkFile.uri}"; File.fromUri(outputFile.resolve("$outputFilename.html.js")).writeAsStringSync( @@ -173,7 +173,7 @@ void createHtmlWrapper(File sdkJsFile, Uri outputFile, String jsContent, jsRootDart, "/root_build/$outputFilename.html.js")); print("You should now be able to run\n\n" - "dart ${sdkRoot.path}/tools/testing/dart/http_server.dart -p 39550 " + "dart ${sdkRoot.path}/pkg/test_runner/bin/http_server.dart -p 39550 " "--network 127.0.0.1 " "--build-directory=${outDir.toFilePath()}" "\n\nand go to\n\n" diff --git a/pkg/pkg.status b/pkg/pkg.status index 367542c0f02..df75de52414 100644 --- a/pkg/pkg.status +++ b/pkg/pkg.status @@ -84,6 +84,7 @@ status_file/test/normalize_test: SkipByDesign # Uses dart:io status_file/test/parse_and_normalize_test: SkipByDesign # Uses dart:io status_file/test/repo_status_files_test: SkipByDesign # Uses dart:io telemetry/test/*: SkipByDesign # Only meant to run on vm +test_runner/test/*: SkipByDesign # Only meant to run on vm testing/*: SkipByDesign # Only meant to run on vm typed_data/test/typed_buffers_test/01: Fail # Not supporting Int64List, Uint64List. diff --git a/pkg/smith/pubspec.yaml b/pkg/smith/pubspec.yaml index 42a702f9b2d..0b80a7ded0e 100644 --- a/pkg/smith/pubspec.yaml +++ b/pkg/smith/pubspec.yaml @@ -2,6 +2,10 @@ name: smith author: Dart Team description: Shared code for working with the Dart SDK's tests and test runner. homepage: http://www.dartlang.org +# This package is not intended to be published. +publish_to: none +environment: + sdk: "^2.3.0" dev_dependencies: expect: path: ../expect diff --git a/pkg/status_file/pubspec.yaml b/pkg/status_file/pubspec.yaml index e3a5960543c..4eb3bccdb5e 100644 --- a/pkg/status_file/pubspec.yaml +++ b/pkg/status_file/pubspec.yaml @@ -1,11 +1,11 @@ name: status_file # This package is not intended to be published. publish_to: none - +environment: + sdk: "^2.3.0" dependencies: path: "^1.4.0" args: "^1.4.4" - dev_dependencies: expect: path: ../expect diff --git a/pkg/status_file/test/data/standalone_2_vm.status b/pkg/status_file/test/data/standalone_2_vm.status index 01c5b89825c..8c521ab6696 100644 --- a/pkg/status_file/test/data/standalone_2_vm.status +++ b/pkg/status_file/test/data/standalone_2_vm.status @@ -35,7 +35,6 @@ no_lazy_dispatchers_test: SkipByDesign # SIMDBC interpreter doesn't support --no io/web_socket_test: Pass, RuntimeError # Issue 26814. [ $system == windows ] -io/skipping_dart2js_compilations_test: Skip # Issue 19551. verbose_gc_to_bmu_test: Skip io/process_sync_test: Pass, Timeout # Issue 24596 io/sleep_test: Pass, Fail # Issue 25757 diff --git a/pkg/test_runner/.gitignore b/pkg/test_runner/.gitignore new file mode 100644 index 00000000000..b26da7f84b9 --- /dev/null +++ b/pkg/test_runner/.gitignore @@ -0,0 +1,3 @@ +# Don’t commit the following files and directories created by pub. +.packages +pubspec.lock diff --git a/pkg/test_runner/bin/http_server.dart b/pkg/test_runner/bin/http_server.dart new file mode 100644 index 00000000000..298e91401c8 --- /dev/null +++ b/pkg/test_runner/bin/http_server.dart @@ -0,0 +1,53 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:test_runner/src/configuration.dart'; +import 'package:test_runner/src/testing_servers.dart'; +import 'package:test_runner/src/utils.dart'; +import 'package:test_runner/src/vendored_pkg/args/args.dart'; + +void main(List arguments) { + var parser = new ArgParser(); + parser.addOption('port', + abbr: 'p', + help: 'The main server port we wish to respond to requests.', + defaultsTo: '0'); + parser.addOption('crossOriginPort', + abbr: 'c', + help: 'A different port that accepts request from the main server port.', + defaultsTo: '0'); + parser.addFlag('help', + abbr: 'h', negatable: false, help: 'Print this usage information.'); + parser.addOption('build-directory', help: 'The build directory to use.'); + parser.addOption('package-root', help: 'The package root to use.'); + parser.addOption('packages', help: 'The package spec file to use.'); + parser.addOption('network', + help: 'The network interface to use.', defaultsTo: '0.0.0.0'); + parser.addFlag('csp', + help: 'Use Content Security Policy restrictions.', defaultsTo: false); + parser.addOption('runtime', + help: 'The runtime we are using (for csp flags).', defaultsTo: 'none'); + + var args = parser.parse(arguments); + if (args['help'] as bool) { + print(parser.getUsage()); + } else { + var servers = new TestingServers( + args['build-directory'] as String, + args['csp'] as bool, + Runtime.find(args['runtime'] as String), + null, + args['package-root'] as String, + args['packages'] as String); + var port = int.parse(args['port'] as String); + var crossOriginPort = int.parse(args['crossOriginPort'] as String); + servers + .startServers(args['network'] as String, + port: port, crossOriginPort: crossOriginPort) + .then((_) { + DebugLogger.info('Server listening on port ${servers.port}'); + DebugLogger.info('Server listening on port ${servers.crossOriginPort}'); + }); + } +} diff --git a/tools/testing/dart/launch_browser.dart b/pkg/test_runner/bin/launch_browser.dart similarity index 91% rename from tools/testing/dart/launch_browser.dart rename to pkg/test_runner/bin/launch_browser.dart index b4f598e7dd2..91dfd9e53a2 100644 --- a/tools/testing/dart/launch_browser.dart +++ b/pkg/test_runner/bin/launch_browser.dart @@ -10,8 +10,8 @@ * DARTBIN should be the checked in stable binary. */ -import 'browser_controller.dart'; -import 'configuration.dart'; +import 'package:test_runner/src/browser_controller.dart'; +import 'package:test_runner/src/configuration.dart'; void printHelp() { print("Usage pattern:"); diff --git a/tools/testing/dart/package_testing_support.dart b/pkg/test_runner/bin/package_testing_support.dart similarity index 69% rename from tools/testing/dart/package_testing_support.dart rename to pkg/test_runner/bin/package_testing_support.dart index 99876a24546..ce9f760d1c6 100644 --- a/tools/testing/dart/package_testing_support.dart +++ b/pkg/test_runner/bin/package_testing_support.dart @@ -2,10 +2,10 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE.md file. -import 'configuration.dart'; -import 'options.dart'; -import 'repository.dart'; -import 'test_configurations.dart'; +import 'package:test_runner/src/configuration.dart'; +import 'package:test_runner/src/options.dart'; +import 'package:test_runner/src/repository.dart'; +import 'package:test_runner/src/test_configurations.dart'; void main(List arguments) { Repository.uri = Uri.base; diff --git a/tools/testing/dart/main.dart b/pkg/test_runner/bin/test_runner.dart similarity index 93% rename from tools/testing/dart/main.dart rename to pkg/test_runner/bin/test_runner.dart index 3e1b904c1bf..2dbad82184e 100644 --- a/tools/testing/dart/main.dart +++ b/pkg/test_runner/bin/test_runner.dart @@ -21,8 +21,8 @@ /// /// The default test directory layout is documented in "test_suite.dart", above /// `factory StandardTestSuite.forDirectory`. -import "options.dart"; -import "test_configurations.dart"; +import "package:test_runner/src/options.dart"; +import "package:test_runner/src/test_configurations.dart"; /// Runs all of the tests specified by the given command line [arguments]. void main(List arguments) { diff --git a/tools/testing/dart/android.dart b/pkg/test_runner/lib/src/android.dart similarity index 100% rename from tools/testing/dart/android.dart rename to pkg/test_runner/lib/src/android.dart diff --git a/tools/testing/dart/babel_transform.js b/pkg/test_runner/lib/src/babel_transform.js similarity index 100% rename from tools/testing/dart/babel_transform.js rename to pkg/test_runner/lib/src/babel_transform.js diff --git a/tools/testing/dart/browser_test.dart b/pkg/test_runner/lib/src/browser.dart similarity index 98% rename from tools/testing/dart/browser_test.dart rename to pkg/test_runner/lib/src/browser.dart index b7f51ca7b7e..c30516bb435 100644 --- a/tools/testing/dart/browser_test.dart +++ b/pkg/test_runner/lib/src/browser.dart @@ -23,7 +23,7 @@ String dart2jsHtml(String title, String scriptPath) {

Running $title

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/constructor_calls_created_synchronously_test.html b/tests/lib_2/html/custom/constructor_calls_created_synchronously_test.html index c18be7d326d..e77e7c7e250 100644 --- a/tests/lib_2/html/custom/constructor_calls_created_synchronously_test.html +++ b/tests/lib_2/html/custom/constructor_calls_created_synchronously_test.html @@ -16,7 +16,7 @@

Running entered_left_view_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/created_callback_test.html b/tests/lib_2/html/custom/created_callback_test.html index 892277b8e49..ca6e193a85f 100644 --- a/tests/lib_2/html/custom/created_callback_test.html +++ b/tests/lib_2/html/custom/created_callback_test.html @@ -16,7 +16,7 @@

Running created_callback_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/document_register_basic_test.html b/tests/lib_2/html/custom/document_register_basic_test.html index 10041c979ae..c4042b3d168 100644 --- a/tests/lib_2/html/custom/document_register_basic_test.html +++ b/tests/lib_2/html/custom/document_register_basic_test.html @@ -16,7 +16,7 @@

Running document_register_basic_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/document_register_template_test.html b/tests/lib_2/html/custom/document_register_template_test.html index 26f59bf77ce..7c48c6a392e 100644 --- a/tests/lib_2/html/custom/document_register_template_test.html +++ b/tests/lib_2/html/custom/document_register_template_test.html @@ -16,7 +16,7 @@

Running document_register_basic_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/document_register_type_extensions_test.html b/tests/lib_2/html/custom/document_register_type_extensions_test.html index 75419cb6681..30c2138ffa7 100644 --- a/tests/lib_2/html/custom/document_register_type_extensions_test.html +++ b/tests/lib_2/html/custom/document_register_type_extensions_test.html @@ -16,7 +16,7 @@

Running document_register_type_extensions_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/element_upgrade_test.html b/tests/lib_2/html/custom/element_upgrade_test.html index 9ae03d34092..e06cdbc9a67 100644 --- a/tests/lib_2/html/custom/element_upgrade_test.html +++ b/tests/lib_2/html/custom/element_upgrade_test.html @@ -44,6 +44,6 @@ function validateIsFoo(element) { + src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/entered_left_view_test.html b/tests/lib_2/html/custom/entered_left_view_test.html index c18be7d326d..e77e7c7e250 100644 --- a/tests/lib_2/html/custom/entered_left_view_test.html +++ b/tests/lib_2/html/custom/entered_left_view_test.html @@ -16,7 +16,7 @@

Running entered_left_view_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/js_custom_test.html b/tests/lib_2/html/custom/js_custom_test.html index 18e254c3eb3..323f54ced10 100644 --- a/tests/lib_2/html/custom/js_custom_test.html +++ b/tests/lib_2/html/custom/js_custom_test.html @@ -16,7 +16,7 @@

Running js_custom_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/mirrors_2_test.html b/tests/lib_2/html/custom/mirrors_2_test.html index bab6ba30cae..225759becca 100644 --- a/tests/lib_2/html/custom/mirrors_2_test.html +++ b/tests/lib_2/html/custom/mirrors_2_test.html @@ -16,7 +16,7 @@

Running mirrors_2_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom/mirrors_test.html b/tests/lib_2/html/custom/mirrors_test.html index 76079b35039..34d84b9c334 100644 --- a/tests/lib_2/html/custom/mirrors_test.html +++ b/tests/lib_2/html/custom/mirrors_test.html @@ -16,7 +16,7 @@

Running mirrors_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom_element_method_clash_test.html b/tests/lib_2/html/custom_element_method_clash_test.html index a5e9625bddb..1c216a502a9 100644 --- a/tests/lib_2/html/custom_element_method_clash_test.html +++ b/tests/lib_2/html/custom_element_method_clash_test.html @@ -16,7 +16,7 @@

Running custom_element_method_clash_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom_element_name_clash_test.html b/tests/lib_2/html/custom_element_name_clash_test.html index f414794c1fe..289b371310e 100644 --- a/tests/lib_2/html/custom_element_name_clash_test.html +++ b/tests/lib_2/html/custom_element_name_clash_test.html @@ -16,7 +16,7 @@

Running custom_element_name_clash_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom_elements_23127_test.html b/tests/lib_2/html/custom_elements_23127_test.html index b0b68b5ef24..1384fa0cf7f 100644 --- a/tests/lib_2/html/custom_elements_23127_test.html +++ b/tests/lib_2/html/custom_elements_23127_test.html @@ -16,7 +16,7 @@

Running custom_elements_23127_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/custom_elements_test.html b/tests/lib_2/html/custom_elements_test.html index 42c6ed6e90f..a5751b8981e 100644 --- a/tests/lib_2/html/custom_elements_test.html +++ b/tests/lib_2/html/custom_elements_test.html @@ -16,7 +16,7 @@

Running custom_elements_test

+ src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/js_dispatch_property_test.html b/tests/lib_2/html/js_dispatch_property_test.html index ba98d98924f..630ed9507c6 100644 --- a/tests/lib_2/html/js_dispatch_property_test.html +++ b/tests/lib_2/html/js_dispatch_property_test.html @@ -21,7 +21,7 @@ + src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/lib_2/html/js_interop_constructor_name_test.html b/tests/lib_2/html/js_interop_constructor_name_test.html index 4b21d8bcab1..04399e4cade 100644 --- a/tests/lib_2/html/js_interop_constructor_name_test.html +++ b/tests/lib_2/html/js_interop_constructor_name_test.html @@ -16,7 +16,7 @@ + src="/root_dart/pkg/test_runner/lib/src/test_controller.js"> %TEST_SCRIPTS% diff --git a/tests/standalone_2/io/test_harness_analyzer_test.dart b/tests/standalone_2/io/test_harness_analyzer_test.dart deleted file mode 100644 index 350cbd9971e..00000000000 --- a/tests/standalone_2/io/test_harness_analyzer_test.dart +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import '../../../tools/testing/dart/main.dart' as test_dart; -import '../../../tools/testing/dart/launch_browser.dart' as launch_browser; - -// The purpose of this test is to run the analyzer on it and make sure our -// testing scripts are free of warnings/errors. - -void main() {} diff --git a/tests/standalone_2/io/test_runner_exit_code_script.dart b/tests/standalone_2/io/test_runner_exit_code_script.dart deleted file mode 100644 index e29843084b9..00000000000 --- a/tests/standalone_2/io/test_runner_exit_code_script.dart +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -// Simulates a use of test_progress during a failing run of test.dart. - -import "dart:io"; -import "../../../tools/testing/dart/test_progress.dart"; -import "../../../tools/testing/dart/test_runner.dart"; -import "../../../tools/testing/dart/test_options.dart"; - -main(List arguments) { - var progressType = arguments[0]; - // Build a progress indicator. - var startTime = new DateTime.now(); - var progress = new ProgressIndicator.fromName(progressType, startTime, false); - if (progressType == 'buildbot') { - BuildbotProgressIndicator.stepName = 'myStepName'; - } - // Build a dummy test case. - var configuration = new TestOptionsParser().parse(['--timeout', '2'])[0]; - var dummyCommand = new Command("noop", []); - var testCase = new TestCase('failing_test.dart', [dummyCommand], - configuration, (_) => null, new Set.from(['PASS'])); - - // Simulate the test.dart use of the progress indicator. - progress.testAdded(); - progress.allTestsKnown(); - progress.start(testCase); - new CommandOutput.fromCase(testCase, dummyCommand, 1, false, false, [], [], - new DateTime.now().difference(startTime), false); - progress.done(testCase); - progress.allDone(); -} diff --git a/tests/standalone_2/standalone_2.status b/tests/standalone_2/standalone_2.status index d229107f1c1..559025fd898 100644 --- a/tests/standalone_2/standalone_2.status +++ b/tests/standalone_2/standalone_2.status @@ -134,7 +134,5 @@ io/addlatexhash_test: Pass, Crash # Issue 31252 io/many_directory_operations_test: SkipSlow io/many_file_operations_test: SkipSlow io/raw_datagram_read_all_test: Pass, Fail # Timing dependent. -io/skipping_dart2js_compilations_test: SkipSlow -io/test_runner_test: SkipSlow package/*: SkipByDesign # Launches VMs in interesting ways. typed_data_isolate_test: SkipSlow diff --git a/tests/standalone_2/standalone_2_kernel.status b/tests/standalone_2/standalone_2_kernel.status index dabc68fa4e9..5a8f8447e69 100644 --- a/tests/standalone_2/standalone_2_kernel.status +++ b/tests/standalone_2/standalone_2_kernel.status @@ -39,7 +39,6 @@ no_lazy_dispatchers_test: SkipByDesign # KBC interpreter doesn't support --no_la [ $compiler == dartkp ] io/arguments_test: Fail # Test harness passes runtime arguments to the compiler -io/test_runner_test: SkipByDesign # Is not relevant for AOT. [ $system == android ] entrypoints_verification_test: Skip # Requires shared objects which the test script doesn't "adb push". @@ -73,7 +72,6 @@ map_insert_remove_oom_test: Skip # Heap limit too low. [ $compiler == dartkp && $mode == debug && $runtime == dart_precompiled ] io/raw_socket_test: Crash -io/skipping_dart2js_compilations_test: Crash io/socket_exception_test: Pass, Crash io/socket_finalizer_test: Pass, Crash io/socket_info_ipv4_test: Pass, Crash diff --git a/tests/standalone_2/standalone_2_precompiled.status b/tests/standalone_2/standalone_2_precompiled.status index e17acac1319..38d1e2cda61 100644 --- a/tests/standalone_2/standalone_2_precompiled.status +++ b/tests/standalone_2/standalone_2_precompiled.status @@ -43,7 +43,6 @@ io/regress_7191_test: Skip io/regress_7679_test: Skip io/secure_unauthorized_test: Skip io/signals_test: Skip -io/skipping_dart2js_compilations_test: RuntimeError # Issue 30008 io/stdin_sync_test: Skip io/stdio_implicit_close_test: Skip io/stdio_nonblocking_test: Skip @@ -60,7 +59,6 @@ dwarf_stack_trace_test: Pass, RuntimeError # Results will flake due to identical [ $runtime == dart_precompiled && $checked ] io/namespace_test: RuntimeError -io/test_runner_test: RuntimeError [ $compiler == app_jit || $compiler == precompiler ] io/compile_all_test: Skip # Incompatible flag --compile_all diff --git a/tests/standalone_2/standalone_2_vm.status b/tests/standalone_2/standalone_2_vm.status index ee8622a38d0..e77465f81b7 100644 --- a/tests/standalone_2/standalone_2_vm.status +++ b/tests/standalone_2/standalone_2_vm.status @@ -18,12 +18,8 @@ io/platform_resolved_executable_test/05: RuntimeError # Issue 33168 io/platform_test: Skip # Platform.executable io/test_extension_fail_test: Skip # Platform.executable io/test_extension_test: Skip # Platform.executable -io/test_runner_test: RuntimeError # Issue 33168 regress_26031_test: Skip # Platform.resolvedExecutable -[ $runtime == vm ] -io/test_runner_test: Skip # Spawns a process which runs in Dart2 mode. - [ $system == android ] io/file_stat_test: Skip # Issue 26376 io/file_system_watcher_test: Skip # Issue 26376 @@ -58,7 +54,6 @@ io/socket_upgrade_to_secure_test: Skip # Issue 27638 [ $system == windows ] io/process_sync_test: Pass, Timeout # Issue 24596 -io/skipping_dart2js_compilations_test: Skip # Issue 19551. io/sleep_test: Pass, Fail # Issue 25757 io/socket_info_ipv6_test: Skip verbose_gc_to_bmu_test: Skip @@ -99,7 +94,6 @@ io/process_sync_test: Timeout, Pass io/dart_std_io_pipe_test: Timeout, Pass io/http_client_stays_alive_test: Skip # Spawns process in Dart2 mode. io/process_sync_test: Timeout, Pass -io/skipping_dart2js_compilations_test: Skip # Spawns process in Dart2 mode. [ $arch == simdbc || $arch == simdbc64 ] full_coverage_test: Skip # TODO(vegorov) SIMDBC interpreter doesn't support coverage yet. diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 9eb3373e9aa..159d8125945 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -192,6 +192,7 @@ "pkg/pkg.status", "pkg/smith/", "pkg/status_file/", + "pkg/test_runner/", "pkg/vm/", "runtime/", "sdk/", @@ -1777,6 +1778,11 @@ "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer", "arguments": ["--fatal-warnings", "pkg/telemetry"] }, + { + "name": "analyze pkg/test_runner", + "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer", + "arguments": ["--fatal-warnings", "pkg/test_runner"] + }, { "name": "analyze pkg/testing", "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer", @@ -1801,12 +1807,6 @@ "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer", "arguments": ["--fatal-warnings", "tools/gardening"] }, - { - "name": "analyze tools/testing/dart", - "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer", - "arguments": ["--fatal-warnings", "tools/testing/dart"] - }, - { "name": "dartanalyzer --batch tests", "arguments": [ diff --git a/tools/status_clean.dart b/tools/status_clean.dart deleted file mode 100644 index 7d8bd2f3ae1..00000000000 --- a/tools/status_clean.dart +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -library status_clean; - -import "dart:async"; -import "dart:convert" show json, utf8; -import "dart:io"; -import "testing/dart/multitest.dart"; -import "testing/dart/status_file_parser.dart"; -import "testing/dart/test_suite.dart" - show - multiHtmlTestGroupRegExp, - multiTestRegExp, - multiHtmlTestRegExp, - TestUtils; -import "testing/dart/utils.dart" show Path; - -// [STATUS_TUPLES] is a list of (suite-name, directory, status-file)-tuples. -final STATUS_TUPLES = [ - ["corelib_2", "tests/corelib_2", "tests/corelib_2/corelib_2.status"], - ["standalone", "tests/standalone", "tests/standalone/standalone.status"], - ["pkg", "pkg", "pkg/pkg.status"], - ["utils", "tests/utils", "tests/utils/utils.status"], - ["samples", "samples", "samples/samples.status"], - ["analyze_library", "sdk", "tests/lib_2/analyzer/analyze_library.status"], - [ - "dart2js_extra", - "tests/compiler/dart2js_extra", - "tests/compiler/dart2js_extra/dart2js_extra.status" - ], - [ - "dart2js_native", - "tests/compiler/dart2js_native", - "tests/compiler/dart2js_native/dart2js_native.status" - ], - [ - "dart2js", - "tests/compiler/dart2js", - "tests/compiler/dart2js/dart2js.status" - ], - [ - "benchmark_smoke", - "tests/benchmark_smoke", - "tests/benchmark_smoke/benchmark_smoke.status" - ], -]; - -void main(List args) { - TestUtils.setDartDirUri(Platform.script.resolve('..')); - usage() { - print("Usage: ${Platform.executable} "); - exit(1); - } - - if (args.length == 0) usage(); - - if (args[0] == 'deflake') { - run(new StatusFileDeflaker()); - } else if (args[0] == 'remove-nonexistent-tests') { - run(new StatusFileNonExistentTestRemover()); - } else { - usage(); - } -} - -run(StatusFileProcessor processor) { - Future.forEach(STATUS_TUPLES, (List tuple) { - String suiteName = tuple[0]; - String directory = tuple[1]; - String filePath = tuple[2]; - print("Processing $filePath"); - return processor.run(suiteName, directory, filePath); - }); -} - -abstract class StatusFileProcessor { - Future run(String suiteName, String directory, String filePath); - - Future> _readSections(String filePath) { - File file = new File(filePath); - - if (file.existsSync()) { - var completer = new Completer(); - List
sections = new List
(); - - ReadConfigurationInto(new Path(file.path), sections, () { - completer.complete(sections); - }); - return completer.future; - } - return new Future.value([]); - } -} - -class StatusFileNonExistentTestRemover extends StatusFileProcessor { - final MultiTestDetector multiTestDetector = new MultiTestDetector(); - final TestFileLister testFileLister = new TestFileLister(); - - Future run(String suiteName, String directory, String filePath) { - return _readSections(filePath).then((List
sections) { - Set invalidLines = _analyzeStatusFile(directory, filePath, sections); - if (invalidLines.length > 0) { - return _writeFixedStatusFile(filePath, invalidLines); - } - return new Future.value(); - }); - } - - bool _testExists(String filePath, List testFiles, String directory, - TestRule rule) { - // TODO: Unify this regular expression matching with status_file_parser.dart - List getRuleRegex(String name) { - return name - .split("/") - .map((name) => new RegExp(name.replaceAll('*', '.*'))) - .toList(); - } - - bool matchRegexp(List patterns, String str) { - var parts = str.split("/"); - if (patterns.length > parts.length) { - return false; - } - // NOTE: patterns.length <= parts.length - for (var i = 0; i < patterns.length; i++) { - if (!patterns[i].hasMatch(parts[i])) { - return false; - } - } - return true; - } - - var rulePattern = getRuleRegex(rule.name); - return testFiles.any((String file) { - // TODO: Use test_suite.dart's [buildTestCaseDisplayName] instead. - var filePath = new Path(file).relativeTo(new Path(directory)); - String baseTestName = _concat( - "${filePath.directoryPath}", "${filePath.filenameWithoutExtension}"); - - List testNames = []; - for (var name in multiTestDetector.getMultitestNames(file)) { - testNames.add(_concat(baseTestName, name)); - } - - // If it is not a multitest the testname is [baseTestName] - if (testNames.isEmpty) { - testNames.add(baseTestName); - } - - return testNames - .any((String testName) => matchRegexp(rulePattern, testName)); - }); - } - - Set _analyzeStatusFile( - String directory, String filePath, List
sections) { - var invalidLines = new Set(); - var dartFiles = testFileLister.listTestFiles(directory); - for (var section in sections) { - for (var rule in section.testRules) { - if (!_testExists(filePath, dartFiles, directory, rule)) { - print("Invalid rule: ${rule.name} in file " - "$filePath:${rule.lineNumber}"); - invalidLines.add(rule.lineNumber); - } - } - } - return invalidLines; - } - - _writeFixedStatusFile(String statusFilePath, Set invalidLines) { - var lines = new File(statusFilePath).readAsLinesSync(); - var outputLines = []; - for (int i = 0; i < lines.length; i++) { - // The status file parser numbers lines starting with 1, not 0. - if (!invalidLines.contains(i + 1)) { - outputLines.add(lines[i]); - } - } - var outputFile = new File("$statusFilePath.fixed"); - outputFile.writeAsStringSync(outputLines.join("\n")); - } - - String _concat(String base, String part) { - if (base == "") return part; - if (part == "") return base; - return "$base/$part"; - } -} - -class StatusFileDeflaker extends StatusFileProcessor { - TestOutcomeFetcher _testOutcomeFetcher = new TestOutcomeFetcher(); - - Future run(String suiteName, String directory, String filePath) { - return _readSections(filePath).then((List
sections) { - return _generatedDeflakedLines(suiteName, sections) - .then((Map fixedLines) { - if (fixedLines.length > 0) { - return _writeFixedStatusFile(filePath, fixedLines); - } - }); - }); - } - - Future _generatedDeflakedLines(String suiteName, List
sections) { - var fixedLines = new Map(); - return Future.forEach(sections, (Section section) { - return Future.forEach(section.testRules, (rule) { - return _maybeFixStatusfileLine(suiteName, section, rule, fixedLines); - }); - }).then((_) => fixedLines); - } - - Future _maybeFixStatusfileLine(String suiteName, Section section, - TestRule rule, Map fixedLines) { - print("Processing ${section.statusFile.location}: ${rule.lineNumber}"); - // None of our status file lines have expressions, so we pass {} here. - var notedOutcomes = rule.expression - .evaluate({}) - .map((name) => Expectation.byName(name)) - .where((Expectation expectation) => !expectation.isMetaExpectation) - .toSet(); - - if (notedOutcomes.isEmpty) return new Future.value(); - - // TODO: [rule.name] is actually a pattern not just a testname. We should - // find all possible testnames this rule matches against and unify the - // outcomes of these tests. - return _testOutcomeFetcher - .outcomesOf(suiteName, section, rule.name) - .then((Set actualOutcomes) { - var outcomesThatNeverHappened = new Set(); - for (Expectation notedOutcome in notedOutcomes) { - bool found = false; - for (Expectation actualOutcome in actualOutcomes) { - if (actualOutcome.canBeOutcomeOf(notedOutcome)) { - found = true; - break; - } - } - if (!found) { - outcomesThatNeverHappened.add(notedOutcome); - } - } - - if (outcomesThatNeverHappened.length > 0 && actualOutcomes.length > 0) { - // Print the change to stdout. - print("${rule.name} " - "(${section.statusFile.location}:${rule.lineNumber}):"); - print(" Actual outcomes: ${actualOutcomes.toList()}"); - print(" Outcomes in status file: ${notedOutcomes.toList()}"); - print(" Outcomes in status file that never happened : " - "${outcomesThatNeverHappened.toList()}\n"); - - // Build the fixed status file line. - fixedLines[rule.lineNumber] = - '${rule.name}: ${actualOutcomes.join(', ')} ' - '# before: ${notedOutcomes.join(', ')} / ' - 'never happened: ${outcomesThatNeverHappened.join(', ')}'; - } - }); - } - - _writeFixedStatusFile(String filePath, Map fixedLines) { - var lines = new File(filePath).readAsLinesSync(); - var outputLines = []; - for (int i = 0; i < lines.length; i++) { - if (fixedLines.containsKey(i + 1)) { - outputLines.add(fixedLines[i + 1]); - } else { - outputLines.add(lines[i]); - } - } - var output = outputLines.join("\n"); - var outputFile = new File("$filePath.deflaked"); - outputFile.writeAsStringSync(output); - } -} - -class MultiTestDetector { - final multiTestsCache = new Map>(); - final multiHtmlTestsCache = new Map>(); - - List getMultitestNames(String file) { - List names = []; - names.addAll(getStandardMultitestNames(file)); - names.addAll(getHtmlMultitestNames(file)); - return names; - } - - List getStandardMultitestNames(String file) { - return multiTestsCache.putIfAbsent(file, () { - try { - var tests = new Map(); - var outcomes = new Map>(); - if (multiTestRegExp.hasMatch(new File(file).readAsStringSync())) { - extractTestsFromMultitest(new Path(file), tests, outcomes); - } - return tests.keys.toList(); - } catch (error) { - print("WARNING: Couldn't determine multitests in file ${file}: $error"); - return []; - } - }); - } - - List getHtmlMultitestNames(String file) { - return multiHtmlTestsCache.putIfAbsent(file, () { - try { - List subtestNames = []; - var content = new File(file).readAsStringSync(); - - if (multiHtmlTestRegExp.hasMatch(content)) { - var matchesIter = - multiHtmlTestGroupRegExp.allMatches(content).iterator; - while (matchesIter.moveNext()) { - String fullMatch = matchesIter.current.group(0); - subtestNames.add(fullMatch.substring(fullMatch.indexOf("'") + 1)); - } - } - return subtestNames; - } catch (error) { - print("WARNING: Couldn't determine multitests in file ${file}: $error"); - } - return []; - }); - } -} - -class TestFileLister { - final Map> _filesCache = {}; - - List listTestFiles(String directory) { - return _filesCache.putIfAbsent(directory, () { - var dir = new Directory(directory); - // Cannot test for _test.dart because co19 tests don't have that ending. - var dartFiles = dir - .listSync(recursive: true) - .where((fe) => fe is File) - .where((file) => - file.path.endsWith(".dart") || file.path.endsWith("_test.html")) - .map((file) => file.path) - .toList(); - return dartFiles; - }); - } -} - -/* - * [TestOutcomeFetcher] will fetch test results from a server using a REST-like - * interface. - */ -class TestOutcomeFetcher { - static String SERVER = '108.170.219.8'; - static int PORT = 4540; - - HttpClient _client = new HttpClient(); - - Future> outcomesOf( - String suiteName, Section section, String testName) { - var pathComponents = [ - 'json', - 'test-outcomes', - 'outcomes', - Uri.encodeComponent("$suiteName/$testName") - ]; - var path = pathComponents.join('/') + '/'; - var url = new Uri(scheme: 'http', host: SERVER, port: PORT, path: path); - - return _client - .getUrl(url) - .then((HttpClientRequest request) => request.close()) - .then((HttpClientResponse response) { - return response - .transform(utf8.decoder) - .transform(json.decoder) - .first - .then((List testResults) { - var setOfActualOutcomes = new Set(); - - try { - for (var result in testResults) { - var config = result['configuration']; - var testResult = result['test_result']; - var outcome = testResult['outcome']; - - // These variables are derived variables and will be set in - // tools/testing/dart/test_options.dart. - // [Mostly due to the fact that we don't have an unary ! - // operator in status file expressions.] - config['unchecked'] = !config['checked']; - config['unminified'] = !config['minified']; - config['nocsp'] = !config['csp']; - config['browser'] = TestUtils.isBrowserRuntime(config['runtime']); - config['analyzer'] = - TestUtils.isCommandLineAnalyzer(config['compiler']); - config['jscl'] = - TestUtils.isJsCommandLineRuntime(config['runtime']); - - if (section.condition == null || - section.condition.evaluate(config)) { - setOfActualOutcomes.add(Expectation.byName(outcome)); - } - } - return setOfActualOutcomes; - } catch (error) { - print("Warning: Error occured while processing testoutcomes" - ": $error"); - return []; - } - }).catchError((error) { - print("Warning: Error occured while fetching testoutcomes: $error"); - return []; - }); - }); - } -} diff --git a/tools/test.py b/tools/test.py index a847a35fe08..18f40e364fd 100755 --- a/tools/test.py +++ b/tools/test.py @@ -13,9 +13,11 @@ import utils def Main(): args = sys.argv[1:] + tools_dir = os.path.dirname(os.path.realpath(__file__)) - dart_test_script = string.join( - [tools_dir, 'testing', 'dart', 'main.dart'], os.sep) + repo_dir = os.path.dirname(tools_dir) + dart_test_script = os.path.join( + repo_dir, 'pkg', 'test_runner', 'bin', 'test_runner.dart') command = [utils.CheckedInSdkExecutable(), dart_test_script] + args # The testing script potentially needs the android platform tools in PATH so diff --git a/tools/testing/dart/analysis_options.yaml b/tools/testing/dart/analysis_options.yaml deleted file mode 100644 index 1a46de2ee05..00000000000 --- a/tools/testing/dart/analysis_options.yaml +++ /dev/null @@ -1,3 +0,0 @@ -analyzer: - strong-mode: - implicit-casts: false diff --git a/tools/testing/dart/pubspec.yaml b/tools/testing/dart/pubspec.yaml deleted file mode 100644 index 4ddbba8b111..00000000000 --- a/tools/testing/dart/pubspec.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file -# for details. All rights reserved. Use of this source code is governed by a -# BSD-style license that can be found in the LICENSE file. - -# This file is only here that so certain Dart editors recognize this is a Dart -# project. -name: testing_tools diff --git a/tools/utils.py b/tools/utils.py index 2d0f6ecc8fd..25bd8e79da9 100644 --- a/tools/utils.py +++ b/tools/utils.py @@ -817,7 +817,8 @@ def TryUnlink(file): class BaseCoreDumpArchiver(object): """This class reads coredumps file written by UnexpectedCrashDumpArchiver into the current working directory and uploads all cores and binaries - listed in it into Cloud Storage (see tools/testing/dart/test_progress.dart). + listed in it into Cloud Storage (see + pkg/test_runner/lib/src/test_progress.dart). """ # test.dart will write a line for each unexpected crash into this file.