From f6205e001376a672f97568a3636c6eae2f7b71b2 Mon Sep 17 00:00:00 2001 From: Ben Konyi Date: Fri, 8 Dec 2023 15:58:52 +0000 Subject: [PATCH] [ test matrix ] Run package:vm_service tests on VM configurations Change-Id: I69eb125f0df46bdc3be65b99985bfcdaa0d51bfe Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/337300 Reviewed-by: Alexander Thomas Commit-Queue: Ben Konyi Reviewed-by: Ryan Macnak --- pkg/pkg.status | 115 +++++++++++- pkg/test_runner/lib/src/options.dart | 1 - pkg/test_runner/test/options_test.dart | 1 - .../test/common/service_test_common.dart | 13 ++ pkg/vm_service/test/common/test_helper.dart | 6 +- .../test/get_cpu_samples_rpc_test.dart | 6 +- .../get_perfetto_cpu_samples_rpc_test.dart | 19 +- .../test/heap_snapshot_graph_test.dart | 2 + .../http_get_isolate_group_rpc_common.dart | 5 +- .../http_get_isolate_rpc_common.dart | 5 +- .../http_get_vm_rpc_common.dart | 5 +- tools/bots/test_matrix.json | 176 ++++++++++++++++-- 12 files changed, 314 insertions(+), 40 deletions(-) diff --git a/pkg/pkg.status b/pkg/pkg.status index 6952757e78b..da54dff0c1e 100644 --- a/pkg/pkg.status +++ b/pkg/pkg.status @@ -86,6 +86,7 @@ analyzer_plugin/test/*: SkipByDesign # Only meant to run on vm analyzer_plugin/tool/*: SkipByDesign # Only meant to run on vm build_integration/test/*: SkipByDesign # Only meant to run on vm, most use dart:mirrors and dart:io compiler/tool/*: SkipByDesign # Only meant to run on vm +dart_service_protocol_shared/test/*: SkipByDesign # Only meant to run on vm dartdev/test/*: SkipByDesign # Only meant to run on vm front_end/test/*: SkipByDesign # Only meant to run on vm, most use dart:mirrors and dart:io front_end/tool/*: SkipByDesign # Only meant to run on vm @@ -93,9 +94,6 @@ modular_test/test/memory_pipeline_test: Slow, Pass modular_test/test/validate_pipeline_test: Slow, Pass modular_test/test/validate_suite_test: Slow, Pass native_assets_builder/test/*: SkipByDesign # Only meant to run on vm -nnbd_migration/test/*: SkipByDesign # Uses mirrors -nnbd_migration/tool/*: SkipByDesign # Only meant to run on vm -dart_service_protocol_shared/test/*: SkipByDesign # Only meant to run on vm smith/test/*: SkipByDesign # Only meant to run on vm status_file/test/normalize_test: SkipByDesign # Uses dart:io status_file/test/parse_and_normalize_test: SkipByDesign # Uses dart:io @@ -105,6 +103,102 @@ test_runner/test/*: SkipByDesign # Only meant to run on vm testing/*: SkipByDesign # Only meant to run on vm wasm/*: SkipByDesign # Only meant to run on vm +[ $compiler == dartkp ] +vm_service/test/*breakpoint*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/*debugger*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/*step_into*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/*step_out*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/async_next_regression_18877_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/async_next_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/async_scope_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/async_single_step_exception_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/awaiter_async_stack_contents_2_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/awaiter_async_stack_contents_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/bad_reload_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/branch_coverage_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/break*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/capture_stdio_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/causal_async*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/code_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/complex_reload_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/coverage_async_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/coverage_closure_call_test: SkipByDesign # Debugger and coverage are disabled in AOT mode. +vm_service/test/coverage_const_field_async_closure_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/coverage_leaf_function_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/coverage_optimized_function_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/debug*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/deferred_import_reload_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/developer_extension_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/developer_service_get_isolate_id_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/developer_service_get_object_id_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/enhanced_enum_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/eval_*test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/evaluate_*test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/external_compilation_service_test: SkipByDesign # Spawns a secondary process. +vm_service/test/field_script_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_allocation_samples_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_allocation_traces_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_instances_as_*: SkipByDesign # Debugger is disabled in AOT mode +vm_service/test/get_instances_rpc_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_object_rpc_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_source_report_const_coverage_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_source_report_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_source_report_with_mixin_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_stack_limit_rpc_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_stack_rpc_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/get_stack_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/implicit_getter_setter_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/invoke_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/isolate_lifecycle_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/issue_25465_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/issue_27238_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/issue_27287_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/issue_30555_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/kill_paused_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/library_dependency_test: SkipByDesign # Uses 'dart:mirrors' library. +vm_service/test/local_variable_declaration_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/local_variable_in_awaiter_async_frame_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/logging_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/mark_main_isolate_as_system_isolate_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/mirror_references_test: SkipByDesign # Uses 'dart:mirrors' library. +vm_service/test/mixin_break_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/network_profiling_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/parameters_in_scope_at_entry_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/patterns_local_vars_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/pause_*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/positive_token_pos_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/private_rpcs/breakpoint_gc_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/private_rpcs/reachable_size_test: SkipByDesign # No incremental compiler available. +vm_service/test/process_service_test: SkipByDesign # Spawns a secondary process using Platform.script. +vm_service/test/regress_28443_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_28980_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_34841_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_44588_test: SkipByDesign # Not relevant in AOT mode. +vm_service/test/regress_45684_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_46419_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_46559_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_48279_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/regress_88104_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/reload_sources_rpc_triggers_isolate_reload_event_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/reload_sources_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/resume_shutdown_race_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/rewind*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/sdk_break_with_mixin_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/set_library_debuggable_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/set_sdk_library_debuggable_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/sigquit_starts_service_test: SkipByDesign # Spawns a secondary process using Platform.executable. +vm_service/test/simple_reload_test: SkipByDesign # Hot reload is disabled in AOT mode. +vm_service/test/source_report_libraries_already_compiled_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/source_report_package_filters_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/step_*: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/super_constructor_invocation_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/uri_mappings_lookup_test: SkipByDesign # Uses Platform.script for URI mappings. +vm_service/test/valid_source_locations_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/validate_timer_port_behavior_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/vm_timeline_flags_test: SkipByDesign # Debugger is disabled in AOT mode. +vm_service/test/weak_properties_test: SkipByDesign # Uses 'dart:mirrors' library +vm_service/test/yield_positions_with_finally_test: SkipByDesign # Debugger is disabled in AOT mode. + [ $mode == debug ] compiler/test/deferred/load_graph_segmentation_test: Slow, Pass compiler/test/deferred/load_mapping_test: Slow, Pass @@ -113,8 +207,8 @@ compiler/test/end_to_end/exit_code_test: Slow, Pass vm/test/kernel_front_end_test: SkipSlow vm_snapshot_analysis/test/precompiler_trace_test: SkipSlow -[ $runtime == dart_precompiled ] -*: SkipByDesign # The pkg test framework imports dart:mirrors. +[ $mode == product ] +vm_service/test*: SkipByDesign # VM service isn't supported in PRODUCT mode. [ $runtime == vm ] analysis_server/test/benchmarks_test: Slow, Pass @@ -143,7 +237,6 @@ dds/test/dap/integration/*: Slow, Pass dart2js_info/test/*: SkipByDesign # Only meant to run on vm dds/test/*: SkipByDesign # Only meant to run on vm dev_compiler/test/options/*: SkipByDesign -front_end/test/hot_reload_e2e_test: Skip frontend_server/test/*: SkipByDesign # Only meant to run on vm js_runtime/test/*: SkipByDesign # Only meant to run on vm js_shared/test/*: SkipByDesign # Only meant to run on vm @@ -151,13 +244,15 @@ linter/test/*: SkipByDesign # Only meant to run on vm linter/tool/*: SkipByDesign # Only meant to run on vm native_stack_traces/test/*: SkipByDesign # Only meant to run on vm vm/test/*: SkipByDesign # Only meant to run on vm -vm_service/test/*: SkipByDesign # Uses dart:io vm_snapshot_analysis/test/*: SkipByDesign # Only meant to run on vm [ $system == windows ] front_end/test/fasta/bootstrap_test: Skip # Issue 31902 front_end/test/fasta/strong_test: Pass, Slow, Timeout front_end/test/incremental_dart2js_load_from_dill_test: Pass, Slow +vm_service/test/private_rpcs/dev_fs_http_put_test: Skip # Windows disallows "?" in paths +vm_service/test/private_rpcs/dev_fs_http_put_weird_char_test: Skip # Windows disallows "\r" in paths +vm_service/test/private_rpcs/dev_fs_weird_char_test: Skip # Windows disallows "\r" in paths [ $browser ] */test/analyzer_test: SkipByDesign # No need to run analysis tests on browser bots @@ -209,6 +304,9 @@ analysis_server/test/*: Skip # Issue 26813 analyzer/test/*: Skip # Issue 26813 analyzer/tool/*: Skip # Issue 26813 +[ $compiler != dart2analyzer && $runtime != dart_precompiled && $runtime != vm ] +vm_service/test/*: SkipByDesign # Only meant to run on vm + [ $compiler != dart2analyzer && $runtime != vm ] dev_compiler/test/*: Skip modular_test/test/find_sdk_root1_test: SkipByDesign @@ -249,9 +347,6 @@ analysis_server/test/analysis/get_errors_test: Skip # runtime error, Issue 22180 analysis_server/test/src/plugin/plugin_manager_test: Slow, Pass # Issue 34231 analyzer/test/src/task/strong/checker_test: Slow, Pass analyzer/tool/task_dependency_graph/check_test: Slow, Pass -vm_service/test/private_rpcs/dev_fs_http_put_test: Skip # Windows disallows "?" in paths -vm_service/test/private_rpcs/dev_fs_http_put_weird_char_test: Skip # Windows disallows "\r" in paths -vm_service/test/private_rpcs/dev_fs_weird_char_test: Skip # Windows disallows "\r" in paths [ $runtime == vm && $system == windows && $checked ] front_end/tool/perf_test: Slow, Pass diff --git a/pkg/test_runner/lib/src/options.dart b/pkg/test_runner/lib/src/options.dart index d30654007ba..1cbe554cd02 100644 --- a/pkg/test_runner/lib/src/options.dart +++ b/pkg/test_runner/lib/src/options.dart @@ -22,7 +22,6 @@ const _defaultTestSelectors = [ 'language', 'lib', 'samples', - 'service', 'standalone', 'utils', 'vm', diff --git a/pkg/test_runner/test/options_test.dart b/pkg/test_runner/test/options_test.dart index 2b8bf4811a8..c8c1e93df73 100644 --- a/pkg/test_runner/test/options_test.dart +++ b/pkg/test_runner/test/options_test.dart @@ -118,7 +118,6 @@ void testSelectors() { 'vm', 'utils', 'lib', - 'service', 'kernel', 'ffi', }, configuration.selectors.keys, "suites for $arguments"); diff --git a/pkg/vm_service/test/common/service_test_common.dart b/pkg/vm_service/test/common/service_test_common.dart index c5ed4e697fc..d458c17615d 100644 --- a/pkg/vm_service/test/common/service_test_common.dart +++ b/pkg/vm_service/test/common/service_test_common.dart @@ -5,6 +5,7 @@ library service_test_common; import 'dart:async'; +import 'dart:developer'; import 'dart:typed_data'; import 'package:path/path.dart'; @@ -517,6 +518,18 @@ List removeAdjacentDuplicates(List fromList) { return result; } +/// Waits for ServiceProtocolInfo.serverUri to be populated. +Future waitForServiceInfo() async { + print('Waiting for the VM service URI to become available...'); + var info = await Service.getInfo(); + while (info.serverUri == null) { + await Future.delayed(const Duration(milliseconds: 100)); + info = await Service.getInfo(); + } + print('VM service URI has become available: ${info.serverUri}'); + return info; +} + typedef ServiceExtensionHandler = Future> Function( Map cb, ); diff --git a/pkg/vm_service/test/common/test_helper.dart b/pkg/vm_service/test/common/test_helper.dart index 08c958e2969..6cb587b7a60 100644 --- a/pkg/vm_service/test/common/test_helper.dart +++ b/pkg/vm_service/test/common/test_helper.dart @@ -35,6 +35,10 @@ Uri _getTestUri(String script) { // If running from pub we can assume that we're in the root of the package // directory. return Uri.parse('test/$script'); + } else if (io.Platform.script.toFilePath().endsWith('out.aotsnapshot')) { + // We're running an AOT test. In this case, we need to use the exact URI we + // launched with. + return io.Platform.script; } else { // Resolve the script to ensure that test will fail if the provided script // name doesn't match the actual script. @@ -167,7 +171,7 @@ class _ServiceTesteeLauncher { Future _spawnCommon( String executable, - List /*!*/ arguments, + List arguments, Map dartEnvironment, ) { final environment = _TESTEE_SPAWN_ENV; diff --git a/pkg/vm_service/test/get_cpu_samples_rpc_test.dart b/pkg/vm_service/test/get_cpu_samples_rpc_test.dart index ed58b052f77..c9a9faa592b 100644 --- a/pkg/vm_service/test/get_cpu_samples_rpc_test.dart +++ b/pkg/vm_service/test/get_cpu_samples_rpc_test.dart @@ -15,7 +15,7 @@ int fib(n) { void testeeDo() { print('Testee doing something.'); - fib(30); + fib(44); print('Testee did something.'); } @@ -34,7 +34,7 @@ Future checkSamples(VmService service, IsolateRef isolate) async { ); final samples = result.samples!; - expect(samples.length, greaterThan(10), reason: 'Should have many samples'); + expect(samples.length, greaterThan(0), reason: 'Should have samples'); expect(samples.length, result.sampleCount); final sample = samples.first; @@ -55,6 +55,8 @@ final tests = [ const vmArgs = [ '--profiler=true', + // Crank up the sampling rate to make sure we get samples. + '--profile_period=100', '--profile-vm=false', // So this also works with KBC. ]; diff --git a/pkg/vm_service/test/get_perfetto_cpu_samples_rpc_test.dart b/pkg/vm_service/test/get_perfetto_cpu_samples_rpc_test.dart index 0d2d03b1fc0..591da600024 100644 --- a/pkg/vm_service/test/get_perfetto_cpu_samples_rpc_test.dart +++ b/pkg/vm_service/test/get_perfetto_cpu_samples_rpc_test.dart @@ -10,6 +10,18 @@ import 'package:vm_service_protos/vm_service_protos.dart'; import 'common/test_helper.dart'; +int fib(n) { + if (n < 0) return 0; + if (n == 0) return 1; + return fib(n - 1) + fib(n - 2); +} + +void testeeDo() { + print('Testee doing something.'); + fib(44); + print('Testee did something.'); +} + int computeTimeOriginNanos(List packets) { final packetsWithPerfSamples = packets.where((packet) => packet.hasPerfSample()).toList(); @@ -102,5 +114,10 @@ void main([args = const []]) => runIsolateTests( args, tests, 'get_perfetto_cpu_samples_rpc_test.dart', - extraArgs: ['--profiler=true'], + testeeBefore: testeeDo, + extraArgs: [ + '--profiler=true', + // Crank up the sampling rate to make sure we get samples. + '--profile_period=100', + ], ); diff --git a/pkg/vm_service/test/heap_snapshot_graph_test.dart b/pkg/vm_service/test/heap_snapshot_graph_test.dart index 8f1a127d802..deaac488095 100644 --- a/pkg/vm_service/test/heap_snapshot_graph_test.dart +++ b/pkg/vm_service/test/heap_snapshot_graph_test.dart @@ -8,7 +8,9 @@ import 'package:vm_service/vm_service.dart'; import 'common/test_helper.dart'; class Foo { + @pragma('vm:entry-point') dynamic left; + @pragma('vm:entry-point') dynamic right; } diff --git a/pkg/vm_service/test/http_invocations/http_get_isolate_group_rpc_common.dart b/pkg/vm_service/test/http_invocations/http_get_isolate_group_rpc_common.dart index 9649a7ef9d9..b9b2227de6a 100644 --- a/pkg/vm_service/test/http_invocations/http_get_isolate_group_rpc_common.dart +++ b/pkg/vm_service/test/http_invocations/http_get_isolate_group_rpc_common.dart @@ -3,13 +3,12 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:developer'; import 'package:test/test.dart'; import 'package:vm_service/vm_service.dart'; import '../common/expect.dart'; -import '../common/test_helper.dart'; +import '../common/service_test_common.dart'; import 'http_request_helpers.dart'; Future getIsolateGroupId( @@ -23,7 +22,7 @@ Future getIsolateGroupId( } Future testeeBefore() async { - final info = await Service.getInfo(); + final info = await waitForServiceInfo(); final serverUri = info.serverUri!; try { final result = createServiceObject( diff --git a/pkg/vm_service/test/http_invocations/http_get_isolate_rpc_common.dart b/pkg/vm_service/test/http_invocations/http_get_isolate_rpc_common.dart index 32370438486..aba91bdba8b 100644 --- a/pkg/vm_service/test/http_invocations/http_get_isolate_rpc_common.dart +++ b/pkg/vm_service/test/http_invocations/http_get_isolate_rpc_common.dart @@ -3,12 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:developer'; import 'package:vm_service/vm_service.dart'; import '../common/expect.dart'; -import '../common/test_helper.dart'; +import '../common/service_test_common.dart'; import 'http_request_helpers.dart'; Future getIsolateId(Uri serverUri) async { @@ -20,7 +19,7 @@ Future getIsolateId(Uri serverUri) async { } Future testeeBefore() async { - final info = await Service.getInfo(); + final info = await waitForServiceInfo(); final serverUri = info.serverUri!; try { diff --git a/pkg/vm_service/test/http_invocations/http_get_vm_rpc_common.dart b/pkg/vm_service/test/http_invocations/http_get_vm_rpc_common.dart index 87dc021c77c..d0771d0e980 100644 --- a/pkg/vm_service/test/http_invocations/http_get_vm_rpc_common.dart +++ b/pkg/vm_service/test/http_invocations/http_get_vm_rpc_common.dart @@ -3,16 +3,15 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:async'; -import 'dart:developer'; import 'package:vm_service/vm_service.dart'; import '../common/expect.dart'; -import '../common/test_helper.dart'; +import '../common/service_test_common.dart'; import 'http_request_helpers.dart'; Future testeeBefore() async { - final info = await Service.getInfo(); + final info = await waitForServiceInfo(); final serverUri = info.serverUri!; try { diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json index cbff748575a..8e10c891814 100644 --- a/tools/bots/test_matrix.json +++ b/tools/bots/test_matrix.json @@ -940,25 +940,84 @@ } ] }, + { + "builders": [ + "vm-linux-debug-x64" + ], + "meta": { + "description": "This configuration is used by the VM JIT builders. Includes VM service testing in addtion to legacy Observatory service tests.", + "__comment__": "This configuration should be merged with the other VM JIT builders config once Observatory is removed." + }, + "steps": [ + { + "name": "build dart", + "script": "tools/build.py", + "arguments": [ + "--codesigning-identity=-", + "runtime" + ] + }, + { + "name": "vm tests", + "arguments": [ + "-nvm-${system}-${mode}-${arch}", + "--default-suites", + "co19", + "pkg/pkg/vm_service/", + "service" + ], + "fileset": "vm", + "shards": 16 + } + ] + }, { "builders": [ "vm-linux-debug-ia32", - "vm-linux-debug-simriscv64", - "vm-linux-debug-x64", - "vm-linux-debug-x64c", "vm-linux-release-ia32", - "vm-linux-release-simarm", - "vm-linux-release-simarm64", "vm-linux-release-x64", "vm-mac-debug-arm64", - "vm-mac-debug-x64", "vm-mac-release-arm64", - "vm-mac-release-x64", "vm-win-debug-x64", - "vm-win-debug-x64c", "vm-win-release-ia32", "vm-win-release-x64" ], + "meta": { + "description": "This configuration is used by the VM JIT builders. Includes VM service testing.", + "__comment__": "vm-linux-debug-x64 should be added back into this configuration once Observatory is removed." + }, + "steps": [ + { + "name": "build dart", + "script": "tools/build.py", + "arguments": [ + "--codesigning-identity=-", + "runtime" + ] + }, + { + "name": "vm tests", + "arguments": [ + "-nvm-${system}-${mode}-${arch}", + "--default-suites", + "co19", + "pkg/pkg/vm_service/" + ], + "fileset": "vm", + "shards": 16 + } + ] + }, + { + "builders": [ + "vm-linux-debug-simriscv64", + "vm-linux-debug-x64c", + "vm-linux-release-simarm", + "vm-linux-release-simarm64", + "vm-mac-debug-x64", + "vm-mac-release-x64", + "vm-win-debug-x64c" + ], "meta": { "description": "This configuration is used by the VM JIT builders." }, @@ -983,17 +1042,78 @@ } ] }, + { + "builders": [ + "vm-aot-linux-debug-x64" + ], + "meta": { + "description": "This configuration is used by the VM AOT builders. Includes VM service testing in addition to legacy Observatory service tests.", + "__comment__": "This configuration should be merged with the other VM AOT builders config below once Observatory is removed." + + }, + "steps": [ + { + "name": "build dart", + "script": "tools/build.py", + "arguments": [ + "runtime", + "dart_precompiled_runtime" + ] + }, + { + "name": "vm tests", + "arguments": [ + "-nvm-aot-${system}-${mode}-${arch}", + "--default-suites", + "co19", + "pkg/pkg/vm_service/", + "service" + ], + "fileset": "vm", + "shards": 20 + } + ] + }, + { + "builders": [ + "vm-aot-linux-release-x64", + "vm-aot-win-release-x64" + ], + "meta": { + "description": "This configuration is used by the VM AOT builders. Includes VM service testing.", + "__comment__": "vm-aot-linux-debug-x64 should be added back into this configuration once Observatory is removed." + + }, + "steps": [ + { + "name": "build dart", + "script": "tools/build.py", + "arguments": [ + "runtime", + "dart_precompiled_runtime" + ] + }, + { + "name": "vm tests", + "arguments": [ + "-nvm-aot-${system}-${mode}-${arch}", + "--default-suites", + "co19", + "pkg/pkg/vm_service/" + ], + "fileset": "vm", + "shards": 20 + } + ] + }, { "builders": [ "vm-aot-linux-debug-simriscv64", - "vm-aot-linux-debug-x64", "vm-aot-linux-debug-x64c", "vm-aot-linux-product-x64", "vm-aot-linux-release-simarm64", - "vm-aot-linux-release-x64", "vm-aot-win-debug-x64c", - "vm-aot-win-product-x64", - "vm-aot-win-release-x64" + "vm-aot-win-product-x64" ], "meta": { "description": "This configuration is used by the VM AOT builders." @@ -1056,10 +1176,36 @@ }, { "builders": [ - "vm-aot-mac-product-arm64", "vm-aot-mac-release-arm64", "vm-aot-mac-release-x64" ], + "meta": { + "description": "This configuration is used by the VM AOT builders. Without shards because shards are missing XCode. Includes VM service testing." + }, + "steps": [ + { + "name": "build dart", + "script": "tools/build.py", + "arguments": [ + "--codesigning-identity=-", + "runtime", + "dart_precompiled_runtime" + ] + }, + { + "name": "vm tests", + "arguments": [ + "-nvm-aot-${system}-${mode}-${arch}", + "--default-suites", + "pkg/pkg/vm_service/" + ] + } + ] + }, + { + "builders": [ + "vm-aot-mac-product-arm64" + ], "meta": { "description": "This configuration is used by the VM AOT builders. Without shards because shards are missing XCode." }, @@ -2827,7 +2973,7 @@ "name": "package unit tests", "arguments": [ "-nunittest-asserts-${mode}-${system}-${arch}", - "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|nnbd_migration|dartdev/test/native_assets)/)" + "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|nnbd_migration|dartdev/test/native_assets|vm_service)/)" ], "shards": 3, "fileset": "vm" @@ -2947,7 +3093,7 @@ "name": "package unit tests", "arguments": [ "-nunittest-asserts-${mode}-${system}-${arch}", - "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|nnbd_migration|dartdev/test/native_assets)/)" + "pkg/pkg/(?!(analyzer*|analysis_server|compiler|dev_compiler|js_runtime|front_end|kernel|nnbd_migration|dartdev/test/native_assets|vm_service)/)" ] } ]