mirror of
https://github.com/dart-lang/sdk
synced 2024-09-19 21:11:40 +00:00
d358abfd1f
Many tests under `pkg`, like `pkg/js_ast` and `pkg/analyzer_cli` are meant to run as unit tests in the Dart VM. These were also being run in dart2js and causing flakiness because they took a long time to build on our CQ. Locally some of these tests compile in under 5s, but we see the compile-time vary among the bots a lot. I've observed cases close to 30s and some that reach the 1-minute timeout cutoff. Flakiness data show that this is happening between 5-7% of the runs, especially on linux bots. This CL stops running these tests on the dart2js configuration by changing which subset of the `pkg` suite is provided to dart2js. It also adjusts the .status file to skip a few suites for convenience on local testing only. This allows developers locally to run `test.py -n... pkg` without worrying about filtering out skipped suites. Addendum: after discussions on the CL we decided not to cmpletely drop coverage of js_ast tests on JS to ensure it can be used in browser environments in the future. To support this we made made the tests cheaper by removing the dependency on package:test. They will now only be run in a single configuration, which is all we need for this purpose. We can reevaluate and remove this if we continue to see timeouts. Change-Id: Idf0dbdd37e412ef71ba117ec979cb1e52585c431 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/330704 Reviewed-by: Stephen Adams <sra@google.com> Commit-Queue: Sigmund Cherem <sigmund@google.com>
269 lines
13 KiB
Plaintext
269 lines
13 KiB
Plaintext
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
# BSD-style license that can be found in the LICENSE file.
|
|
# Don't run any test-like files that show up in packages directories. It
|
|
# shouldn't be necessary to run "pub install" in these packages, but if you do
|
|
# it shouldn't break the tests.
|
|
*/*/*/*/*/packages/*/*: Skip
|
|
*/*/*/*/packages/*/*: Skip
|
|
*/*/*/packages/*/*: Skip
|
|
*/*/packages/*/*: Skip
|
|
*/packages/*/*: Skip
|
|
analyzer/test/src/task/strong/checker_test: Slow, Pass
|
|
analyzer/test/verify_diagnostics_test.dart: Slow, Pass
|
|
analyzer/test/verify_docs_test.dart: Slow, Pass
|
|
analyzer_plugin/test/plugin/folding_mixin_test: Slow, Pass
|
|
compiler/test/analyses/analyze_test: Slow, Pass
|
|
compiler/test/analyses/api_dynamic_test: Slow, Pass
|
|
compiler/test/analyses/dart2js_dynamic_test: Slow, Pass
|
|
compiler/test/closure/closure_test: Slow, Pass
|
|
compiler/test/codegen/load_elimination_test: Slow, Pass
|
|
compiler/test/codegen/model_test: Slow, Pass
|
|
compiler/test/deferred_loading/deferred_loading_test: Slow, Pass
|
|
compiler/test/end_to_end/dump_info_test: Skip #47401
|
|
compiler/test/impact/impact_test: Slow, Pass
|
|
compiler/test/inference/inference0_test: Slow, Pass
|
|
compiler/test/inference/inference1_test: Slow, Pass
|
|
compiler/test/inference/inference2_test: Slow, Pass
|
|
compiler/test/inference/inference3_test: Slow, Pass
|
|
compiler/test/inlining/inlining_test: Slow, Pass
|
|
compiler/test/model/native_test: Slow, Pass
|
|
compiler/test/model/no_such_method_enabled_test: Slow, Pass
|
|
compiler/test/model/subtype_test: Slow, Pass
|
|
compiler/test/modular/*: Slow, Pass
|
|
compiler/test/packages/*: SkipByDesign
|
|
compiler/test/rti/rti_emission0_test: Slow, Pass
|
|
compiler/test/rti/rti_emission1_test: Slow, Pass
|
|
compiler/test/rti/rti_emission2_test: Slow, Pass
|
|
compiler/test/rti/rti_emission3_test: Slow, Pass
|
|
compiler/test/rti/rti_need0_test: Slow, Pass
|
|
compiler/test/rti/rti_need1_test: Slow, Pass
|
|
compiler/test/rti/rti_need2_test: Slow, Pass
|
|
compiler/test/rti/rti_need3_test: Slow, Pass
|
|
compiler/test/serialization/serialization_test: Slow, Pass
|
|
compiler/test/sourcemaps/source_mapping_invokes_test: Slow, Pass
|
|
compiler/test/sourcemaps/source_mapping_operators_test: Slow, Pass
|
|
compiler/test/sourcemaps/source_mapping_test: Slow, Pass
|
|
compiler/test/sourcemaps/stacktrace_test: Slow, Pass
|
|
dartdev/test/commands/analyze_test: Slow, Pass
|
|
dartdev/test/commands/help_test: Slow, Pass
|
|
dartdev/test/smoke/*: Slow, Pass
|
|
dev_compiler/test/modular/*: Slow, Pass
|
|
dev_compiler/test/options/*: Skip # test needs fixes
|
|
dev_compiler/test/sourcemap/*: SkipByDesign # Skip sourcemap tests
|
|
dev_compiler/test/sourcemap/testfiles/*: SkipByDesign # Skip dev_compiler codegen tests
|
|
dev_compiler/test/worker/*: Skip # test needs fixes
|
|
front_end/test/fasta/analyze_git_test: Pass, Slow
|
|
front_end/test/fasta/bootstrap_test: Pass, Slow
|
|
front_end/test/fasta/rasta/*: SkipByDesign # Anything in rasta is input to fasta unit tests and shouldn't be run as tests.
|
|
front_end/test/fasta/sdk_test: SkipByDesign # sdk_test would take too long to complete, and should be run in a different way.
|
|
front_end/test/fasta/types/dart2js_benchmark_test: Pass, Slow
|
|
front_end/test/fasta/types/large_app_benchmark_test: Pass, ExtraSlow
|
|
front_end/test/incremental_compiler_leak_test: Pass, ExtraSlow
|
|
front_end/test/incremental_dart2js_test: Pass, Slow
|
|
front_end/testcases/*: Skip # These are not tests but input for tests.
|
|
front_end/tool/incremental_perf_test: Slow, Pass
|
|
kernel/testcases/*: Skip # These are not tests but input for tests.
|
|
vm/test/kernel_front_end_test: Slow, Pass
|
|
vm/test/transformations/type_flow/transformer_test: Slow, Pass
|
|
vm/testcases/*: SkipByDesign # These are not tests but input for tests.
|
|
vm_service/test/cpu_samples_stream_test: Slow, Pass # Requires CPU sample buffer to fill.
|
|
wasm/*: SkipByDesign # These can't be run without running wasm:setup first.
|
|
|
|
[ $compiler == dart2analyzer ]
|
|
dev_compiler/test/options/*: SkipByDesign
|
|
testing/test/dart_sdk_negative_test: SkipByDesign
|
|
|
|
[ $compiler == dart2js ]
|
|
_fe_analyzer_shared/test/*: SkipByDesign # Only meant to run on vm
|
|
analysis_server/test/integration: SkipByDesign # Analysis server integration tests don't make sense to run under dart2js, since the code under test always runs in the Dart vm as a subprocess.
|
|
analysis_server/tool/*: SkipByDesign # Only meant to run on vm
|
|
analysis_server_client/test/*: SkipByDesign # Only meant to run on vm
|
|
analyzer_cli/test/*: SkipByDesign # Only meant to run on vm
|
|
analyzer_cli/tool/*: SkipByDesign # Only meant to run on vm
|
|
analyzer_plugin/test/*: SkipByDesign # Only meant to run on vm
|
|
analyzer_plugin/tool/*: SkipByDesign # Only meant to run on vm
|
|
build_integration/test/*: SkipByDesign # Only meant to run on vm, most use dart:mirrors and dart:io
|
|
compiler/tool/*: SkipByDesign # Only meant to run on vm
|
|
dartdev/test/*: SkipByDesign # Only meant to run on vm
|
|
front_end/test/*: SkipByDesign # Only meant to run on vm, most use dart:mirrors and dart:io
|
|
front_end/tool/*: SkipByDesign # Only meant to run on vm
|
|
modular_test/test/memory_pipeline_test: Slow, Pass
|
|
modular_test/test/validate_pipeline_test: Slow, Pass
|
|
modular_test/test/validate_suite_test: Slow, Pass
|
|
native_assets_builder/test/*: SkipByDesign # Only meant to run on vm
|
|
nnbd_migration/test/*: SkipByDesign # Uses mirrors
|
|
nnbd_migration/tool/*: SkipByDesign # Only meant to run on vm
|
|
service_extension_router/test/*: SkipByDesign # Only meant to run on vm
|
|
smith/test/*: SkipByDesign # Only meant to run on vm
|
|
status_file/test/normalize_test: SkipByDesign # Uses dart:io
|
|
status_file/test/parse_and_normalize_test: SkipByDesign # Uses dart:io
|
|
status_file/test/repo_status_files_test: SkipByDesign # Uses dart:io
|
|
telemetry/test/*: SkipByDesign # Only meant to run on vm
|
|
test_runner/test/*: SkipByDesign # Only meant to run on vm
|
|
testing/*: SkipByDesign # Only meant to run on vm
|
|
wasm/*: SkipByDesign # Only meant to run on vm
|
|
|
|
[ $mode == debug ]
|
|
compiler/test/deferred/load_graph_segmentation_test: Slow, Pass
|
|
compiler/test/deferred/load_mapping_test: Slow, Pass
|
|
compiler/test/end_to_end/dart2js_batch_test: Slow, Pass
|
|
compiler/test/end_to_end/exit_code_test: Slow, Pass
|
|
vm/test/kernel_front_end_test: SkipSlow
|
|
vm_snapshot_analysis/test/precompiler_trace_test: SkipSlow
|
|
|
|
[ $runtime == dart_precompiled ]
|
|
*: SkipByDesign # The pkg test framework imports dart:mirrors.
|
|
|
|
[ $runtime == vm ]
|
|
analysis_server/test/benchmarks_test: Slow, Pass
|
|
analysis_server/test/domain_completion_test: Slow, Pass
|
|
analysis_server/test/edit/refactoring_test: Slow, Pass
|
|
analysis_server/test/integration/*: Slow, Pass
|
|
analysis_server/test/services/completion/dart/imported_reference_contributor_test: Slow, Pass
|
|
analysis_server/test/services/completion/dart/keyword_contributor_test: Slow, Pass
|
|
analysis_server/test/services/completion/dart/local_reference_contributor_test: Slow, Pass
|
|
analysis_server/test/services/completion/dart/type_member_contributor_test: Slow, Pass
|
|
analysis_server/test/services/refactoring/extract_method_test: Slow, Pass
|
|
analysis_server/test/services/refactoring/inline_method_test: Slow, Pass
|
|
analysis_server/test/socket_server_test: Skip # Pass, Slow
|
|
analysis_server/test/src/plugin/plugin_manager_test: Slow, Pass
|
|
analyzer/test/generated/strong_mode_driver_test: Slow, Pass
|
|
analyzer/test/src/dart/analysis/driver_resolution_kernel_test: Slow, Pass
|
|
analyzer/test/src/dart/analysis/driver_test: Slow, Pass
|
|
analyzer/test/src/task/strong/inferred_type_test: Slow, Pass
|
|
analyzer/tool/experiments/experiments_test: Skip # https://github.com/dart-lang/sdk/issues/46277
|
|
analyzer_plugin/test/src/utilities/change_builder/change_builder_dart_test: Slow, Pass
|
|
analyzer_plugin/test/src/utilities/completion/optype_test: Slow, Pass
|
|
dartdev/test/*: Slow, Pass
|
|
dds/test/dap/integration/*: Slow, Pass
|
|
|
|
[ $runtime != vm ]
|
|
dart2js_info/test/*: SkipByDesign # Only meant to run on vm
|
|
dds/test/*: SkipByDesign # Only meant to run on vm
|
|
dev_compiler/test/options/*: SkipByDesign
|
|
front_end/test/hot_reload_e2e_test: Skip
|
|
frontend_server/test/*: SkipByDesign # Only meant to run on vm
|
|
js_runtime/test/*: SkipByDesign # Only meant to run on vm
|
|
js_shared/test/*: SkipByDesign # Only meant to run on vm
|
|
linter/test/*: SkipByDesign # Only meant to run on vm
|
|
linter/tool/*: SkipByDesign # Only meant to run on vm
|
|
native_stack_traces/test/*: SkipByDesign # Only meant to run on vm
|
|
vm/test/*: SkipByDesign # Only meant to run on vm
|
|
vm_service/test/*: SkipByDesign # Uses dart:io
|
|
vm_snapshot_analysis/test/*: SkipByDesign # Only meant to run on vm
|
|
|
|
[ $system == windows ]
|
|
front_end/test/fasta/bootstrap_test: Skip # Issue 31902
|
|
front_end/test/fasta/strong_test: Pass, Slow, Timeout
|
|
front_end/test/incremental_dart2js_load_from_dill_test: Pass, Slow
|
|
|
|
[ $browser ]
|
|
*/test/analyzer_test: SkipByDesign # No need to run analysis tests on browser bots
|
|
_fe_analyzer_shared/test/*: SkipByDesign # Only meant to run on vm
|
|
analysis_server/test/*: SkipByDesign # Uses dart:io.
|
|
analysis_server/tool/spec/check_all_test: SkipByDesign # Uses dart:io.
|
|
analyzer/test/*: SkipByDesign # Uses dart:io.
|
|
analyzer/tool/summary/check_test: SkipByDesign # Uses dart:io.
|
|
analyzer_cli/*: SkipByDesign # Uses dart:io.
|
|
compiler/tool/*: SkipByDesign # Only meant to run on vm
|
|
front_end/tool/*: SkipByDesign # Only meant to run on vm
|
|
kernel/test/*: SkipByDesign # Uses dart:io and bigints.
|
|
status_file/*: SkipByDesign # Only meant to run on the standalone VM.
|
|
testing/test/analyze_test: SkipByDesign
|
|
|
|
[ $checked ]
|
|
compiler/test/codegen/value_range_test: Slow, Pass
|
|
compiler/test/end_to_end/exit_code_test: Slow, Pass
|
|
compiler/test/end_to_end/output_type_test: Slow, Pass
|
|
compiler/test/end_to_end/uri_retention_test: Slow, Pass
|
|
compiler/test/jsinterop/declaration_test: Slow, Pass
|
|
compiler/test/jsinterop/interop_anonymous_unreachable_test: Slow, Pass
|
|
compiler/test/jsinterop/world_test: Slow, Pass
|
|
compiler/test/sourcemaps/stacktrace_test: Slow, Pass
|
|
|
|
[ !$checked ]
|
|
compiler/test/end_to_end/exit_code_test: SkipByDesign # This tests requires checked mode.
|
|
compiler/test/jsinterop/declaration_test: Slow, Pass
|
|
|
|
[ $jscl ]
|
|
kernel/test/*: SkipByDesign # Uses dart:io and bigints.
|
|
|
|
# Timeout. These tests do not run efficiently on our simulator.
|
|
[ $simulator ]
|
|
*: Skip
|
|
|
|
[ $arch == x64 && $runtime == vm && $system == windows && $checked ]
|
|
analyzer/test/src/task/strong/inferred_type_test: Slow, Pass
|
|
|
|
[ $builder_tag != dart2js_analyzer && $compiler == dart2js ]
|
|
analysis_server/test/*: Skip # Issue 26813
|
|
analyzer/test/*: Skip # Issue 26813
|
|
analyzer/tool/*: Skip # Issue 26813
|
|
|
|
[ $compiler != dart2analyzer && $runtime != vm ]
|
|
dev_compiler/test/*: Skip
|
|
modular_test/test/find_sdk_root1_test: SkipByDesign
|
|
modular_test/test/io_pipeline_test: SkipByDesign
|
|
modular_test/test/loader/loader_test: SkipByDesign
|
|
modular_test/test/specification_parser_test: SkipByDesign
|
|
modular_test/test/src/find_sdk_root2_test: SkipByDesign
|
|
|
|
[ $compiler == dart2js && $runtime == d8 ]
|
|
front_end/test/src/base/uri_resolver_test: SkipByDesign # Relies on access to file system
|
|
|
|
[ $compiler == dart2js && $runtime != d8 ]
|
|
front_end/test/mixin_export_test: RuntimeError # Issue 30576
|
|
|
|
[ $compiler == dart2js && $browser ]
|
|
crypto/test/sha1_test: Slow, Pass
|
|
crypto/test/sha256_test: Slow, Pass
|
|
|
|
[ $compiler == dart2js && $checked ]
|
|
crypto/test/base64_test: Slow, Pass
|
|
|
|
[ $compiler == dart2js && $host_checked ]
|
|
js_ast/test/printer_callback_test: Slow, Pass
|
|
js_ast/test/string_escape_test: Slow, Pass
|
|
|
|
[ $compiler == dart2js && ($runtime == chrome || $runtime == ff) ]
|
|
async/test/stream_zip_test: SkipSlow # Times out. Issue 22050
|
|
collection/test/unmodifiable_collection_test: SkipSlow # Times out. Issue 22050
|
|
|
|
[ $runtime == vm && $system != linux ]
|
|
dartdev/test/commands/info_linux_test: SkipByDesign
|
|
|
|
[ $runtime == vm && $system != macos ]
|
|
dartdev/test/commands/info_macos_test: SkipByDesign
|
|
|
|
[ $runtime == vm && $system == windows ]
|
|
analysis_server/test/analysis/get_errors_test: Skip # runtime error, Issue 22180
|
|
analysis_server/test/src/plugin/plugin_manager_test: Slow, Pass # Issue 34231
|
|
analyzer/test/src/task/strong/checker_test: Slow, Pass
|
|
analyzer/tool/task_dependency_graph/check_test: Slow, Pass
|
|
vm_service/test/private_rpcs/dev_fs_http_put_test: Skip # Windows disallows "?" in paths
|
|
vm_service/test/private_rpcs/dev_fs_http_put_weird_char_test: Skip # Windows disallows "\r" in paths
|
|
vm_service/test/private_rpcs/dev_fs_weird_char_test: Skip # Windows disallows "\r" in paths
|
|
|
|
[ $runtime == vm && $system == windows && $checked ]
|
|
front_end/tool/perf_test: Slow, Pass
|
|
|
|
[ $runtime == vm && $system != windows ]
|
|
dartdev/test/commands/info_windows_test: SkipByDesign
|
|
|
|
[ $runtime == vm && $checked ]
|
|
analysis_server/test/completion_test: Slow, Pass
|
|
analysis_server/test/integration/edit/sort_members_test: Slow, Pass
|
|
analysis_server/test/socket_server_test: Skip # Pass, Slow
|
|
|
|
[ $mode == debug || $runtime != vm || $system == android ]
|
|
vm/test/modular_kernel_plus_aot_test: SkipByDesign # This test should only run if binary is run from build dir
|
|
|
|
[ $mode != release || $runtime != vm ]
|
|
front_end/test/fasta/*: Skip
|
|
front_end/tool/_fasta/*: Skip
|
|
|
|
[ $browser || $jscl ]
|
|
compiler/test/*: Skip # dart2js uses #import('dart:io'); and it is not self-hosted (yet).
|
|
mmap/*: SkipByDesign # Only meant to run on vm
|