diff --git a/pkg/status_file/test/data/standalone_2_vm.status b/pkg/status_file/test/data/standalone_2_vm.status index d58085f65bc..2ce625eb30f 100644 --- a/pkg/status_file/test/data/standalone_2_vm.status +++ b/pkg/status_file/test/data/standalone_2_vm.status @@ -17,7 +17,7 @@ io/regress_7191_test: Pass, Timeout # Issue 28374: timeout. [ $runtime == vm && $system == macos && $mode == release ] io/named_pipe_script_test: Pass, RuntimeError # Issue 28737 -[ $runtime == vm && $system == linux && $mode == release && $arch == ia32 && $builder_tag == asan ] +[ $runtime == vm && $system == linux && $mode == release && $arch == ia32 && $sanitizer == asan ] io/socket_close_test: Pass, Timeout # Issue 28502: timeout. [ $arch == ia32 ] diff --git a/pkg/status_file/test/data/vm.status b/pkg/status_file/test/data/vm.status index d5e0d99afa0..3b59b0fe2d1 100644 --- a/pkg/status_file/test/data/vm.status +++ b/pkg/status_file/test/data/vm.status @@ -131,7 +131,7 @@ dart/spawn_infinite_loop_test: Skip # VM shutdown test dart/spawn_shutdown_test: Skip # VM Shutdown test dart/hello_fuchsia_test: SkipByDesign # This is a test for fuchsia OS -[ $builder_tag == asan ] +[ $sanitizer == asan ] cc/CodeImmutability: Fail,OK # Address Sanitizer turns a crash into a failure. cc/IsolateReload_DanglingGetter_Class: Fail # Issue 28349 cc/IsolateReload_DanglingGetter_Instance: Fail # Issue 28349 diff --git a/pkg/test_runner/lib/src/environment.dart b/pkg/test_runner/lib/src/environment.dart index fe005b573ad..dc170fbe729 100644 --- a/pkg/test_runner/lib/src/environment.dart +++ b/pkg/test_runner/lib/src/environment.dart @@ -33,6 +33,7 @@ final _variables = { "nnbd": _Variable((TestConfiguration c) => c.nnbdMode.name, NnbdMode.names), "qemu": _Variable.bool((c) => c.useQemu), "runtime": _Variable(_runtimeName, _runtimeNames), + "sanitizer": _Variable((c) => c.sanitizer.name, Sanitizer.names), "simulator": _Variable.bool((c) => c.isSimulator), "spec_parser": _Variable.bool((c) => c.compiler == Compiler.specParser), "system": _Variable(_systemName, _systemNames), diff --git a/pkg/test_runner/lib/src/runtime_configuration.dart b/pkg/test_runner/lib/src/runtime_configuration.dart index f3e5973191f..3764f6fcb5b 100644 --- a/pkg/test_runner/lib/src/runtime_configuration.dart +++ b/pkg/test_runner/lib/src/runtime_configuration.dart @@ -320,7 +320,7 @@ class DartVmRuntimeConfiguration extends RuntimeConfiguration { multiplier *= 2; } if (_configuration.sanitizer != Sanitizer.none) { - multiplier *= 2; + multiplier *= 4; } if (_configuration.rr) { multiplier *= 2; diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index 8c252b3bd30..c7d1ac7c401 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -59,10 +59,6 @@ dart_2/disassemble_aot_test: SkipByDesign # IA32 does not support AOT. dart_2/regress32597_2_test: Pass, Slow # Uses --optimization-counter-threshold=10 without a kernel service snapshot. dart_2/regress38467_test: Pass, Slow # Uses --optimization-counter-threshold=10 without a kernel service snapshot. -[ $builder_tag == asan ] -dart/transferable_throws_oom_test: SkipByDesign # This test tries to allocate too much memory on purpose. Still dartbug.com/37188 -dart_2/transferable_throws_oom_test: SkipByDesign # This test tries to allocate too much memory on purpose. Still dartbug.com/37188 - [ $builder_tag == obfuscated ] dart/heapsnapshot_cli_test: SkipByDesign # Relies on symbol names dart/isolate_exit_unsendable_test: SkipByDesign # Relies on symbol names @@ -84,34 +80,6 @@ dart_2/kernel_determinism_test: SkipSlow dart_2/minimal_kernel_test: SkipSlow # gen_kernel is too slow with optimization_counter_threshold dart_2/null_safety_autodetection_in_kernel_compiler_test: SkipSlow # gen_kernel is too slow with optimization_counter_threshold -[ $builder_tag == tsan ] -dart/analyze_snapshot_binary_test: SkipSlow -dart/appjit_cha_deopt_test: SkipSlow -dart/gc/fragmentation*: Skip # Uses too much memory with TSAN overhead and kills bot -dart/gc/out_of_memory*: Skip # Uses too much memory with TSAN overhead and kills bot -dart/gc/scavenger_abort_test: SkipSlow -dart/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. -dart/regress_40462_test: SkipSlow -dart/regress_40753_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. -dart/regress_48523_test: Pass, Slow -dart/trigger_gc_in_native_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. -dart/unobfuscated_static_symbols_test: Pass, Slow # Test includes creating several snapshots in both ELF and assembly modes. -dart/use_strip_flag_test: Pass, Slow # This test can take a longer time to complete. -dart/v8_snapshot_profile_writer_test: SkipSlow -dart/weak_canonical_string_table_test: Pass, Slow # This test can take a longer time to complete. -dart_2/analyze_snapshot_binary_test: SkipSlow -dart_2/appjit_cha_deopt_test: SkipSlow -dart_2/gc/fragmentation*: Skip # Uses too much memory with TSAN overhead and kills bot -dart_2/gc/out_of_memory*: Skip # Uses too much memory with TSAN overhead and kills bot -dart_2/gc/scavenger_abort_test: SkipSlow -dart_2/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. -dart_2/regress_40462_test: SkipSlow -dart_2/regress_40753_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. -dart_2/trigger_gc_in_native_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. -dart_2/use_strip_flag_test: Pass, Slow # This test can take a longer time to complete. -dart_2/v8_snapshot_profile_writer_test: SkipSlow -dart_2/weak_canonical_string_table_test: Pass, Slow # This test can take a longer time to complete. - [ $compiler == app_jitk ] dart/snapshot_version_test: RuntimeError dart_2/snapshot_version_test: RuntimeError @@ -198,6 +166,38 @@ dart_2/reachability_test: SkipByDesign # Developer API not supported in product [ $nnbd == legacy ] dart/*: SkipByDesign # Migrated tests are not supposed to run on non-NNBD bots. +[ $sanitizer == asan ] +dart/transferable_throws_oom_test: SkipByDesign # This test tries to allocate too much memory on purpose. Still dartbug.com/37188 +dart_2/transferable_throws_oom_test: SkipByDesign # This test tries to allocate too much memory on purpose. Still dartbug.com/37188 + +[ $sanitizer == tsan ] +dart/analyze_snapshot_binary_test: SkipSlow +dart/appjit_cha_deopt_test: SkipSlow +dart/gc/fragmentation*: Skip # Uses too much memory with TSAN overhead and kills bot +dart/gc/out_of_memory*: Skip # Uses too much memory with TSAN overhead and kills bot +dart/gc/scavenger_abort_test: SkipSlow +dart/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. +dart/regress_40462_test: SkipSlow +dart/regress_40753_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. +dart/regress_48523_test: Pass, Slow +dart/trigger_gc_in_native_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. +dart/unobfuscated_static_symbols_test: Pass, Slow # Test includes creating several snapshots in both ELF and assembly modes. +dart/use_strip_flag_test: Pass, Slow # This test can take a longer time to complete. +dart/v8_snapshot_profile_writer_test: SkipSlow +dart/weak_canonical_string_table_test: Pass, Slow # This test can take a longer time to complete. +dart_2/analyze_snapshot_binary_test: SkipSlow +dart_2/appjit_cha_deopt_test: SkipSlow +dart_2/gc/fragmentation*: Skip # Uses too much memory with TSAN overhead and kills bot +dart_2/gc/out_of_memory*: Skip # Uses too much memory with TSAN overhead and kills bot +dart_2/gc/scavenger_abort_test: SkipSlow +dart_2/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. +dart_2/regress_40462_test: SkipSlow +dart_2/regress_40753_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. +dart_2/trigger_gc_in_native_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatedly fails to locate its coredump. +dart_2/use_strip_flag_test: Pass, Slow # This test can take a longer time to complete. +dart_2/v8_snapshot_profile_writer_test: SkipSlow +dart_2/weak_canonical_string_table_test: Pass, Slow # This test can take a longer time to complete. + [ $system == android ] dart/isolates/dart_api_create_lightweight_isolate_test: SkipByDesign # On android this test does not work due to not being able to identify library uri. dart/sdk_hash_test: SkipByDesign # The test doesn't know location of cross-platform gen_snapshot @@ -395,14 +395,14 @@ dart/isolates/fast_object_copy_test*: SkipSlow dart_2/awaiter_stacks/async_throws_stack_lazy_non_symbolic_test: Pass, Slow dart_2/isolates/fast_object_copy_test*: SkipSlow -[ $compiler == dartkp && ($builder_tag == tsan || $simulator) ] -dart/regress_45898_test: Pass, Slow -dart_2/regress_45898_test: Pass, Slow - [ $compiler == dartkp && ($runtime == dart_precompiled || $runtime == vm) ] dart/spawn_shutdown_test: SkipSlow dart_2/spawn_shutdown_test: SkipSlow +[ $compiler == dartkp && ($sanitizer == tsan || $simulator) ] +dart/regress_45898_test: Pass, Slow +dart_2/regress_45898_test: Pass, Slow + [ $mode == debug && $system == windows ] dart/spawn_shutdown_test: Skip # Flaky crashes unable to start thread; likely low memory on the bot. dart_2/spawn_shutdown_test: Skip # Flaky crashes unable to start thread; likely low memory on the bot. diff --git a/samples/samples.status b/samples/samples.status index 1c074b410dc..0cea15b080b 100644 --- a/samples/samples.status +++ b/samples/samples.status @@ -2,7 +2,7 @@ # for details. All rights reserved. Use of this source code is governed by a # BSD-style license that can be found in the LICENSE file. -[ $builder_tag == asan ] +[ $sanitizer == asan ] ffi/samples_test: SkipByDesign # FFI skips, see ffi.status [ $simulator ] diff --git a/tests/ffi/ffi.status b/tests/ffi/ffi.status index 681c78929cc..3b909e70cab 100644 --- a/tests/ffi/ffi.status +++ b/tests/ffi/ffi.status @@ -73,13 +73,10 @@ vmspecific_regress_37511_test: SkipByDesign # Symbols are not exposed on purpose vmspecific_regress_37780_test: SkipByDesign # Symbols are not exposed on purpose and are not linked in Windows Precompiled. dartbug.com/40579 vmspecific_regress_51794_test: SkipByDesign # Symbols are not exposed on purpose and are not linked in Windows Precompiled. dartbug.com/40579 -[ $builder_tag == asan || $builder_tag == msan || $builder_tag == tsan ] -data_not_asan_test: SkipByDesign # This test tries to allocate too much memory on purpose. - # These tests trigger and catch an abort (intentionally) and terminate the VM. # They're incompatible with ASAN because not all memory is freed when aborting and # with AppJit because the abort the VM before it can generate a snapshot. -[ $builder_tag == asan || $compiler == app_jitk ] +[ $compiler == app_jitk || $sanitizer == asan ] vmspecific_function_callbacks_exit_test: SkipByDesign [ $compiler == dart2analyzer || $compiler == fasta ] @@ -96,3 +93,6 @@ abi_specific_int_incomplete_jit_test: SkipByDesign # Only intended to run in JIT [ $compiler != dartkp || $arch == arm64 && $system == fuchsia ] abi_specific_int_incomplete_aot_test: SkipByDesign # Only intended to run in AOT mode. + +[ $sanitizer == asan || $sanitizer == msan || $sanitizer == tsan ] +data_not_asan_test: SkipByDesign # This test tries to allocate too much memory on purpose. diff --git a/tests/ffi_2/ffi_2.status b/tests/ffi_2/ffi_2.status index 5b42ec0736a..26a9238776e 100644 --- a/tests/ffi_2/ffi_2.status +++ b/tests/ffi_2/ffi_2.status @@ -60,13 +60,10 @@ vmspecific_handle_test: SkipByDesign # Symbols are not exposed on purpose and ar [ $system != android && $system != linux && $system != macos && $system != windows ] *: Skip # FFI not yet supported on other OSes. -[ $builder_tag == asan || $builder_tag == msan || $builder_tag == tsan ] -data_not_asan_test: SkipByDesign # This test tries to allocate too much memory on purpose. - # These tests trigger and catch an abort (intentionally) and terminate the VM. # They're incompatible with ASAN because not all memory is freed when aborting and # with AppJit because the abort the VM before it can generate a snapshot. -[ $builder_tag == asan || $compiler == app_jitk ] +[ $compiler == app_jitk || $sanitizer == asan ] vmspecific_function_callbacks_exit_test: SkipByDesign [ $compiler == dart2analyzer || $compiler == fasta ] @@ -77,3 +74,6 @@ abi_specific_int_incomplete_jit_test: SkipByDesign # Only intended to run in JIT [ $compiler != dartkp || $arch == arm64 && $system == fuchsia ] abi_specific_int_incomplete_aot_test: SkipByDesign # Only intended to run in AOT mode. + +[ $sanitizer == asan || $sanitizer == msan || $sanitizer == tsan ] +data_not_asan_test: SkipByDesign # This test tries to allocate too much memory on purpose. diff --git a/tests/standalone/standalone.status b/tests/standalone/standalone.status index a0f66e072a7..7e0de639f21 100644 --- a/tests/standalone/standalone.status +++ b/tests/standalone/standalone.status @@ -14,9 +14,6 @@ io/non_utf8_link_test: Skip # Issue 33519. Temp files causing bots to go purple. packages_file_test: Skip # Issue 26715 packages_file_test/none: Skip # contains no tests. -[ $builder_tag == asan ] -io/process_detached_test: Slow, Pass - [ $builder_tag == dwarf ] io/socket_connect_stacktrace_test: SkipByDesign # Assumes stacktrace can be inspected directly, without decoding stack_trace/full1_test: SkipByDesign # Relies on symbolic stack traces. @@ -78,6 +75,9 @@ io/wait_for_event_zone_test: SkipByDesign # Uses mirrors. io/wait_for_test: SkipByDesign # Uses mirrors. verbose_gc_to_bmu_test: Skip # Attempts to spawn dart using Platform.executable +[ $sanitizer == asan ] +io/process_detached_test: Slow, Pass + [ $system != macos ] io/https_connection_closed_during_handshake_test: SkipByDesign # long_ssl_cert_evaluation needed for long handshake is only supported on mac. io/https_nonblocking_trust_evaluation_test: SkipByDesign diff --git a/tests/standalone/standalone_kernel.status b/tests/standalone/standalone_kernel.status index b0002144eda..ca3830c368d 100644 --- a/tests/standalone/standalone_kernel.status +++ b/tests/standalone/standalone_kernel.status @@ -98,11 +98,11 @@ io/web_socket_test: Skip # Flaky. map_insert_remove_oom_test: Skip # Heap limit too low. no_support_debugger_test: Skip # kernel-service snapshot not compatible with flag disabled -[ $builder_tag == asan || $builder_tag == lsan || $builder_tag == msan || $builder_tag == tsan ] -regress_52715_test/measure: Skip # Shadow memory and scudo quarantine interfere with measuring RSS - [ $builder_tag == crossword || $compiler != dartk && $compiler != dartkp || $compiler == dartkp && $system == windows ] entrypoints_verification_test: SkipByDesign # Requires VM to run. Cannot run in precompiled Windows because the DLL is linked against dart.exe instead of dart_precompiled_runtime.exe. Cannot run in cross-word environment as native extension is not built. [ $compiler != dartk || $runtime != vm ] check_for_aot_snapshot_jit_test: SkipByDesign # Test relies on paths, requires JIT test environment. + +[ $sanitizer == asan || $sanitizer == lsan || $sanitizer == msan || $sanitizer == tsan ] +regress_52715_test/measure: Skip # Shadow memory and scudo quarantine interfere with measuring RSS diff --git a/tests/standalone_2/standalone_2.status b/tests/standalone_2/standalone_2.status index fc0adec89de..4edeb994a2f 100644 --- a/tests/standalone_2/standalone_2.status +++ b/tests/standalone_2/standalone_2.status @@ -14,9 +14,6 @@ io/non_utf8_link_test: Skip # Issue 33519. Temp files causing bots to go purple. packages_file_test: Skip # Issue 26715 packages_file_test/none: Skip # contains no tests. -[ $builder_tag == asan ] -io/process_detached_test: Slow, Pass - [ $builder_tag == dwarf ] io/socket_connect_stacktrace_test: SkipByDesign # Assumes stacktrace can be inspected directly, without decoding @@ -71,6 +68,9 @@ io/wait_for_event_zone_test: SkipByDesign # Uses mirrors. io/wait_for_test: SkipByDesign # Uses mirrors. verbose_gc_to_bmu_test: Skip # Attempts to spawn dart using Platform.executable +[ $sanitizer == asan ] +io/process_detached_test: Slow, Pass + # Fuchsia test runner doesn't support multi-tests yet. [ $system == fuchsia ] deny_listed_test: Skip diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index 978e2d61348..048fe1245ab 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -392,65 +392,16 @@ "use-elf": true } }, - "vm-asan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "asan", - "timeout": 240 - } - }, - "vm-lsan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "lsan", - "timeout": 240 - } - }, - "vm-msan-linux-(debug|product|release)-(x64|simarm64|simriscv64)": { - "options": { - "builder-tag": "msan", - "timeout": 240 - } - }, - "vm-tsan-(linux|mac)-(debug|product|release)-(x64|simarm64|simriscv64)": { - "options": { - "builder-tag": "tsan", - "timeout": 240 - } - }, - "vm-ubsan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "ubsan", - "timeout": 240 - } - }, - "vm-aot-asan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "asan" - } - }, - "vm-aot-lsan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "lsan", - "timeout": 240 - } - }, - "vm-aot-msan-linux-(debug|product|release)-(x64|simarm64|simriscv64)": { - "options": { - "builder-tag": "msan", - "timeout": 240 - } - }, - "vm-aot-tsan-(linux|mac)-(debug|product|release)-(x64|simarm64|simriscv64)": { - "options": { - "builder-tag": "tsan", - "timeout": 240 - } - }, - "vm-aot-ubsan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": { - "options": { - "builder-tag": "ubsan", - "timeout": 240 - } - }, + "vm-asan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": {}, + "vm-lsan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": {}, + "vm-msan-linux-(debug|product|release)-(x64|simarm64|simriscv64)": {}, + "vm-tsan-(linux|mac)-(debug|product|release)-(x64|simarm64|simriscv64)": {}, + "vm-ubsan-(linux|mac)-(debug|product|release)-(ia32|x64|simarm|simarm64|simriscv32|simriscv64)": {}, + "vm-aot-asan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": {}, + "vm-aot-lsan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": {}, + "vm-aot-msan-linux-(debug|product|release)-(x64|simarm64|simriscv64)": {}, + "vm-aot-tsan-(linux|mac)-(debug|product|release)-(x64|simarm64|simriscv64)": {}, + "vm-aot-ubsan-(linux|mac)-(debug|product|release)-(x64|simarm|simarm64|simriscv32|simriscv64)": {}, "dart2js-(linux|win)-chrome": { "options": { "use-sdk": true