mirror of
https://github.com/dart-lang/sdk
synced 2024-10-04 18:36:29 +00:00
[CFE] Coverage on try bot
This runs the strong suite, does coverage, and compares the coverage (percentage) to the (hardcoded) expected percentage. Change-Id: I2e7511d7841b19c15139a7c84bf0861b56b81ccc Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/366340 Reviewed-by: Johnni Winther <johnniwinther@google.com> Reviewed-by: Alexander Thomas <athom@google.com> Commit-Queue: Jens Johansen <jensj@google.com>
This commit is contained in:
Jens Johansen
Commit Queue
parent
8b8ec2aa3d
commit
4ab652b38f
506
pkg/front_end/test/coverage_suite.dart
Normal file
506
pkg/front_end/test/coverage_suite.dart
Normal file
|
|
@ -0,0 +1,506 @@
|
|||
// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
import 'dart:convert' show jsonEncode;
|
||||
import 'dart:io'
|
||||
show Directory, File, Platform, Process, ProcessResult, exitCode;
|
||||
|
||||
import 'package:args/args.dart' show ArgParser;
|
||||
import 'package:args/src/arg_results.dart';
|
||||
|
||||
import '../tool/coverage_merger.dart' as coverageMerger;
|
||||
|
||||
bool debug = false;
|
||||
|
||||
// This is the state as of 8b8ec2aa3d91a7dc9e61bd7b9e576f82bf2b52c3,
|
||||
// using out/ReleaseX64/dart-sdk/bin/dart (which for instance makes a
|
||||
// difference for compute_platform_binaries_location.dart).
|
||||
const Map<String, double> _expect = {
|
||||
"package:front_end/src/api_prototype/compiler_options.dart":
|
||||
18.53448275862069,
|
||||
"package:front_end/src/api_prototype/experimental_flags.dart":
|
||||
81.08108108108108,
|
||||
"package:front_end/src/api_prototype/experimental_flags_generated.dart":
|
||||
53.34928229665071,
|
||||
"package:front_end/src/api_prototype/file_system.dart": 33.33333333333333,
|
||||
"package:front_end/src/api_prototype/incremental_kernel_generator.dart":
|
||||
6.666666666666667,
|
||||
"package:front_end/src/api_prototype/kernel_generator.dart": 0.0,
|
||||
"package:front_end/src/api_prototype/language_version.dart": 0.0,
|
||||
"package:front_end/src/api_prototype/lowering_predicates.dart":
|
||||
3.6036036036036037,
|
||||
"package:front_end/src/api_prototype/memory_file_system.dart": 25.0,
|
||||
"package:front_end/src/api_prototype/standard_file_system.dart":
|
||||
38.46153846153847,
|
||||
"package:front_end/src/api_prototype/summary_generator.dart": 0.0,
|
||||
"package:front_end/src/api_prototype/terminal_color_support.dart": 0.0,
|
||||
"package:front_end/src/api_unstable/compiler_state.dart": 0.0,
|
||||
"package:front_end/src/api_unstable/dart2js.dart": 0.0,
|
||||
"package:front_end/src/api_unstable/util.dart": 37.03703703703704,
|
||||
"package:front_end/src/base/instrumentation.dart": 100.0,
|
||||
"package:front_end/src/base/processed_options.dart": 41.36807817589577,
|
||||
"package:front_end/src/compute_platform_binaries_location.dart":
|
||||
64.1025641025641,
|
||||
"package:front_end/src/fasta/builder/builder.dart": 63.63636363636363,
|
||||
"package:front_end/src/fasta/builder/builder_mixins.dart": 100.0,
|
||||
"package:front_end/src/fasta/builder/builtin_type_declaration_builder.dart":
|
||||
70.0,
|
||||
"package:front_end/src/fasta/builder/class_builder.dart": 71.78217821782178,
|
||||
"package:front_end/src/fasta/builder/constructor_reference_builder.dart":
|
||||
100.0,
|
||||
"package:front_end/src/fasta/builder/declaration_builder.dart": 100.0,
|
||||
"package:front_end/src/fasta/builder/dynamic_type_declaration_builder.dart":
|
||||
100.0,
|
||||
"package:front_end/src/fasta/builder/extension_builder.dart": 50.0,
|
||||
"package:front_end/src/fasta/builder/extension_type_declaration_builder.dart":
|
||||
100.0,
|
||||
"package:front_end/src/fasta/builder/fixed_type_builder.dart":
|
||||
22.727272727272727,
|
||||
"package:front_end/src/fasta/builder/formal_parameter_builder.dart":
|
||||
95.85492227979275,
|
||||
"package:front_end/src/fasta/builder/function_type_builder.dart":
|
||||
77.95275590551181,
|
||||
"package:front_end/src/fasta/builder/future_or_type_declaration_builder.dart":
|
||||
0.0,
|
||||
"package:front_end/src/fasta/builder/inferable_type_builder.dart": 100.0,
|
||||
"package:front_end/src/fasta/builder/invalid_type_builder.dart":
|
||||
33.33333333333333,
|
||||
"package:front_end/src/fasta/builder/invalid_type_declaration_builder.dart":
|
||||
85.0,
|
||||
"package:front_end/src/fasta/builder/library_builder.dart": 81.81818181818183,
|
||||
"package:front_end/src/fasta/builder/member_builder.dart": 96.75324675324676,
|
||||
"package:front_end/src/fasta/builder/metadata_builder.dart":
|
||||
86.04651162790698,
|
||||
"package:front_end/src/fasta/builder/mixin_application_builder.dart": 100.0,
|
||||
"package:front_end/src/fasta/builder/modifier_builder.dart":
|
||||
76.47058823529412,
|
||||
"package:front_end/src/fasta/builder/named_type_builder.dart":
|
||||
74.94949494949495,
|
||||
"package:front_end/src/fasta/builder/never_type_declaration_builder.dart":
|
||||
76.92307692307693,
|
||||
"package:front_end/src/fasta/builder/null_type_declaration_builder.dart":
|
||||
33.33333333333333,
|
||||
"package:front_end/src/fasta/builder/nullability_builder.dart": 100.0,
|
||||
"package:front_end/src/fasta/builder/omitted_type_builder.dart":
|
||||
34.146341463414636,
|
||||
"package:front_end/src/fasta/builder/omitted_type_declaration_builder.dart":
|
||||
0.0,
|
||||
"package:front_end/src/fasta/builder/prefix_builder.dart": 89.1891891891892,
|
||||
"package:front_end/src/fasta/builder/record_type_builder.dart":
|
||||
74.35897435897436,
|
||||
"package:front_end/src/fasta/builder/type_alias_builder.dart":
|
||||
78.99159663865547,
|
||||
"package:front_end/src/fasta/builder/type_builder.dart": 77.77777777777779,
|
||||
"package:front_end/src/fasta/builder/type_declaration_builder.dart": 90.0,
|
||||
"package:front_end/src/fasta/builder/type_variable_builder.dart":
|
||||
78.42227378190255,
|
||||
"package:front_end/src/fasta/builder/void_type_declaration_builder.dart":
|
||||
100.0,
|
||||
"package:front_end/src/fasta/builder_graph.dart": 55.769230769230774,
|
||||
"package:front_end/src/fasta/codes/fasta_codes_cfe_generated.dart":
|
||||
68.76132930513596,
|
||||
"package:front_end/src/fasta/codes/type_labeler.dart": 83.68336025848141,
|
||||
"package:front_end/src/fasta/combinator.dart": 100.0,
|
||||
"package:front_end/src/fasta/command_line_reporting.dart": 69.38775510204081,
|
||||
"package:front_end/src/fasta/compiler_context.dart": 90.1639344262295,
|
||||
"package:front_end/src/fasta/configuration.dart": 100.0,
|
||||
"package:front_end/src/fasta/crash.dart": 57.95454545454546,
|
||||
"package:front_end/src/fasta/dill/dill_builder_mixins.dart": 100.0,
|
||||
"package:front_end/src/fasta/dill/dill_class_builder.dart": 92.34972677595628,
|
||||
"package:front_end/src/fasta/dill/dill_extension_builder.dart":
|
||||
86.74698795180723,
|
||||
"package:front_end/src/fasta/dill/dill_extension_member_builder.dart":
|
||||
70.29702970297029,
|
||||
"package:front_end/src/fasta/dill/dill_extension_type_declaration_builder.dart":
|
||||
95.42483660130719,
|
||||
"package:front_end/src/fasta/dill/dill_extension_type_member_builder.dart":
|
||||
83.76623376623377,
|
||||
"package:front_end/src/fasta/dill/dill_library_builder.dart": 78.134110787172,
|
||||
"package:front_end/src/fasta/dill/dill_loader.dart": 85.36585365853658,
|
||||
"package:front_end/src/fasta/dill/dill_member_builder.dart":
|
||||
86.52173913043478,
|
||||
"package:front_end/src/fasta/dill/dill_target.dart": 74.35897435897436,
|
||||
"package:front_end/src/fasta/dill/dill_type_alias_builder.dart":
|
||||
95.83333333333334,
|
||||
"package:front_end/src/fasta/export.dart": 72.22222222222221,
|
||||
"package:front_end/src/fasta/hybrid_file_system.dart": 50.0,
|
||||
"package:front_end/src/fasta/identifiers.dart": 84.78260869565217,
|
||||
"package:front_end/src/fasta/ignored_parser_errors.dart": 100.0,
|
||||
"package:front_end/src/fasta/import.dart": 100.0,
|
||||
"package:front_end/src/fasta/import_chains.dart": 96.96969696969697,
|
||||
"package:front_end/src/fasta/incremental_compiler.dart": 47.94444444444444,
|
||||
"package:front_end/src/fasta/incremental_serializer.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/augmentation_lowering.dart": 100.0,
|
||||
"package:front_end/src/fasta/kernel/benchmarker.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/body_builder.dart": 91.08417508417509,
|
||||
"package:front_end/src/fasta/kernel/body_builder_context.dart":
|
||||
68.31501831501832,
|
||||
"package:front_end/src/fasta/kernel/collections.dart": 36.44736842105264,
|
||||
"package:front_end/src/fasta/kernel/combined_member_signature.dart":
|
||||
91.62790697674419,
|
||||
"package:front_end/src/fasta/kernel/const_conditional_simplifier.dart": 50.0,
|
||||
"package:front_end/src/fasta/kernel/constant_collection_builders.dart":
|
||||
89.23076923076924,
|
||||
"package:front_end/src/fasta/kernel/constant_evaluator.dart":
|
||||
84.91086587436332,
|
||||
"package:front_end/src/fasta/kernel/constant_int_folder.dart":
|
||||
92.04545454545455,
|
||||
"package:front_end/src/fasta/kernel/constructor_tearoff_lowering.dart":
|
||||
95.11278195488721,
|
||||
"package:front_end/src/fasta/kernel/exhaustiveness.dart": 74.57098283931357,
|
||||
"package:front_end/src/fasta/kernel/expression_generator.dart":
|
||||
80.00631711939356,
|
||||
"package:front_end/src/fasta/kernel/expression_generator_helper.dart": 100.0,
|
||||
"package:front_end/src/fasta/kernel/forest.dart": 90.27777777777779,
|
||||
"package:front_end/src/fasta/kernel/forwarding_node.dart": 94.4927536231884,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/class_member.dart":
|
||||
83.1896551724138,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/delayed.dart": 100.0,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/extension_type_members.dart":
|
||||
87.52834467120182,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/hierarchy_builder.dart":
|
||||
50.256410256410255,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/hierarchy_node.dart":
|
||||
93.33333333333333,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/members_builder.dart":
|
||||
98.51851851851852,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/members_node.dart":
|
||||
91.44427001569859,
|
||||
"package:front_end/src/fasta/kernel/hierarchy/mixin_inferrer.dart":
|
||||
61.53846153846154,
|
||||
"package:front_end/src/fasta/kernel/implicit_field_type.dart":
|
||||
52.24719101123596,
|
||||
"package:front_end/src/fasta/kernel/implicit_type_argument.dart":
|
||||
2.941176470588235,
|
||||
"package:front_end/src/fasta/kernel/internal_ast.dart": 46.40625,
|
||||
"package:front_end/src/fasta/kernel/invalid_type.dart": 74.13793103448276,
|
||||
"package:front_end/src/fasta/kernel/kernel_constants.dart": 45.45454545454545,
|
||||
"package:front_end/src/fasta/kernel/kernel_helper.dart": 98.95833333333334,
|
||||
"package:front_end/src/fasta/kernel/kernel_target.dart": 79.56093868281606,
|
||||
"package:front_end/src/fasta/kernel/kernel_variable_builder.dart":
|
||||
61.111111111111114,
|
||||
"package:front_end/src/fasta/kernel/late_lowering.dart": 100.0,
|
||||
"package:front_end/src/fasta/kernel/load_library_builder.dart":
|
||||
87.8048780487805,
|
||||
"package:front_end/src/fasta/kernel/macro/annotation_parser.dart":
|
||||
0.1984126984126984,
|
||||
"package:front_end/src/fasta/kernel/macro/identifiers.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/macro/introspectors.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/macro/macro.dart": 0.22675736961451248,
|
||||
"package:front_end/src/fasta/kernel/macro/offsets.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/macro/types.dart": 0.0,
|
||||
"package:front_end/src/fasta/kernel/member_covariance.dart":
|
||||
89.23611111111111,
|
||||
"package:front_end/src/fasta/kernel/resource_identifier.dart":
|
||||
39.473684210526315,
|
||||
"package:front_end/src/fasta/kernel/static_weak_references.dart":
|
||||
15.238095238095239,
|
||||
"package:front_end/src/fasta/kernel/try_constant_evaluator.dart":
|
||||
19.753086419753085,
|
||||
"package:front_end/src/fasta/kernel/type_algorithms.dart": 93.4560327198364,
|
||||
"package:front_end/src/fasta/kernel/type_builder_computer.dart":
|
||||
90.20618556701031,
|
||||
"package:front_end/src/fasta/kernel/utils.dart": 37.93103448275862,
|
||||
"package:front_end/src/fasta/kernel/verifier.dart": 56.25,
|
||||
"package:front_end/src/fasta/library_graph.dart": 79.3103448275862,
|
||||
"package:front_end/src/fasta/messages.dart": 30.0,
|
||||
"package:front_end/src/fasta/modifier.dart": 100.0,
|
||||
"package:front_end/src/fasta/operator.dart": 100.0,
|
||||
"package:front_end/src/fasta/problems.dart": 0.0,
|
||||
"package:front_end/src/fasta/scope.dart": 80.14256619144604,
|
||||
"package:front_end/src/fasta/source/class_declaration.dart":
|
||||
80.29556650246306,
|
||||
"package:front_end/src/fasta/source/diet_listener.dart": 91.22807017543859,
|
||||
"package:front_end/src/fasta/source/diet_parser.dart": 100.0,
|
||||
"package:front_end/src/fasta/source/name_scheme.dart": 93.19148936170212,
|
||||
"package:front_end/src/fasta/source/outline_builder.dart": 91.54411764705883,
|
||||
"package:front_end/src/fasta/source/redirecting_factory_body.dart":
|
||||
94.44444444444444,
|
||||
"package:front_end/src/fasta/source/source_builder_mixins.dart":
|
||||
88.8268156424581,
|
||||
"package:front_end/src/fasta/source/source_class_builder.dart": 85.9375,
|
||||
"package:front_end/src/fasta/source/source_constructor_builder.dart":
|
||||
92.68817204301075,
|
||||
"package:front_end/src/fasta/source/source_enum_builder.dart":
|
||||
95.75371549893843,
|
||||
"package:front_end/src/fasta/source/source_extension_builder.dart":
|
||||
61.261261261261254,
|
||||
"package:front_end/src/fasta/source/source_extension_type_declaration_builder.dart":
|
||||
85.34136546184739,
|
||||
"package:front_end/src/fasta/source/source_factory_builder.dart":
|
||||
92.27129337539432,
|
||||
"package:front_end/src/fasta/source/source_field_builder.dart":
|
||||
89.18507235338919,
|
||||
"package:front_end/src/fasta/source/source_function_builder.dart":
|
||||
89.36170212765957,
|
||||
"package:front_end/src/fasta/source/source_library_builder.dart":
|
||||
82.6852338413032,
|
||||
"package:front_end/src/fasta/source/source_loader.dart": 78.41158402808249,
|
||||
"package:front_end/src/fasta/source/source_member_builder.dart":
|
||||
40.32258064516129,
|
||||
"package:front_end/src/fasta/source/source_procedure_builder.dart":
|
||||
96.13259668508287,
|
||||
"package:front_end/src/fasta/source/source_type_alias_builder.dart":
|
||||
97.63313609467455,
|
||||
"package:front_end/src/fasta/source/stack_listener_impl.dart":
|
||||
64.44444444444444,
|
||||
"package:front_end/src/fasta/ticker.dart": 100.0,
|
||||
"package:front_end/src/fasta/type_inference/closure_context.dart":
|
||||
83.73015873015873,
|
||||
"package:front_end/src/fasta/type_inference/delayed_expressions.dart":
|
||||
77.55474452554745,
|
||||
"package:front_end/src/fasta/type_inference/external_ast_helper.dart":
|
||||
97.88732394366197,
|
||||
"package:front_end/src/fasta/type_inference/factor_type.dart":
|
||||
76.19047619047619,
|
||||
"package:front_end/src/fasta/type_inference/for_in.dart": 75.47169811320755,
|
||||
"package:front_end/src/fasta/type_inference/inference_results.dart":
|
||||
85.12820512820512,
|
||||
"package:front_end/src/fasta/type_inference/inference_visitor.dart":
|
||||
90.22701475595913,
|
||||
"package:front_end/src/fasta/type_inference/inference_visitor_base.dart":
|
||||
84.06985032074127,
|
||||
"package:front_end/src/fasta/type_inference/matching_cache.dart":
|
||||
80.26509572901325,
|
||||
"package:front_end/src/fasta/type_inference/matching_expressions.dart":
|
||||
98.10964083175804,
|
||||
"package:front_end/src/fasta/type_inference/object_access_target.dart":
|
||||
78.43137254901961,
|
||||
"package:front_end/src/fasta/type_inference/shared_type_analyzer.dart": 98.0,
|
||||
"package:front_end/src/fasta/type_inference/standard_bounds.dart":
|
||||
71.42857142857143,
|
||||
"package:front_end/src/fasta/type_inference/type_constraint_gatherer.dart":
|
||||
61.22715404699739,
|
||||
"package:front_end/src/fasta/type_inference/type_demotion.dart":
|
||||
77.77777777777779,
|
||||
"package:front_end/src/fasta/type_inference/type_inference_engine.dart":
|
||||
86.76975945017182,
|
||||
"package:front_end/src/fasta/type_inference/type_inferrer.dart":
|
||||
51.461988304093566,
|
||||
"package:front_end/src/fasta/type_inference/type_schema.dart":
|
||||
36.666666666666664,
|
||||
"package:front_end/src/fasta/type_inference/type_schema_elimination.dart":
|
||||
88.88888888888889,
|
||||
"package:front_end/src/fasta/type_inference/type_schema_environment.dart":
|
||||
79.50530035335689,
|
||||
"package:front_end/src/fasta/uri_offset.dart": 100.0,
|
||||
"package:front_end/src/fasta/uri_translator.dart": 75.92592592592592,
|
||||
"package:front_end/src/fasta/uris.dart": 100.0,
|
||||
"package:front_end/src/fasta/util/error_reporter_file_copier.dart": 0.0,
|
||||
"package:front_end/src/fasta/util/experiment_environment_getter.dart":
|
||||
85.71428571428571,
|
||||
"package:front_end/src/fasta/util/helpers.dart": 52.63157894736842,
|
||||
"package:front_end/src/fasta/util/parser_ast.dart": 5.567451820128479,
|
||||
"package:front_end/src/fasta/util/parser_ast_helper.dart": 20.424013434089,
|
||||
"package:front_end/src/fasta/util/textual_outline.dart": 86.54205607476636,
|
||||
"package:front_end/src/kernel_generator_impl.dart": 0.5376344086021506,
|
||||
"package:front_end/src/macros/isolate_macro_serializer.dart": 0.0,
|
||||
"package:front_end/src/macros/macro_serializer.dart": 0.0,
|
||||
"package:front_end/src/macros/macro_target.dart": 0.0,
|
||||
"package:front_end/src/macros/macro_target_io.dart": 0.0,
|
||||
"package:front_end/src/macros/temp_dir_macro_serializer.dart": 0.0,
|
||||
};
|
||||
|
||||
Future<void> main([List<String> arguments = const <String>[]]) async {
|
||||
Directory coverageTmpDir =
|
||||
Directory.systemTemp.createTempSync("cfe_coverage");
|
||||
try {
|
||||
await _run(coverageTmpDir, arguments);
|
||||
} finally {
|
||||
if (debug) {
|
||||
print("Data available in $coverageTmpDir");
|
||||
} else {
|
||||
coverageTmpDir.deleteSync(recursive: true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _run(Directory coverageTmpDir, List<String> arguments) async {
|
||||
Stopwatch totalRuntime = new Stopwatch()..start();
|
||||
|
||||
List<String> results = [];
|
||||
List<String> logs = [];
|
||||
Options options = Options.parse(arguments);
|
||||
debug = options.debug;
|
||||
List<Future<ProcessResult>> futures = [];
|
||||
|
||||
if (options.verbose) {
|
||||
print("NOTE: Will run with ${options.numberOfWorkers} shards.");
|
||||
print("");
|
||||
}
|
||||
|
||||
print("Note: Has ${Platform.numberOfProcessors} cores.");
|
||||
|
||||
for (int i = 0; i < options.numberOfWorkers; i++) {
|
||||
print("Starting shard ${i + 1} of ${options.numberOfWorkers}");
|
||||
futures.add(Process.run(Platform.resolvedExecutable, [
|
||||
"--enable-asserts",
|
||||
"pkg/front_end/test/fasta/strong_suite.dart",
|
||||
"-DskipVm=true",
|
||||
"--shards=${options.numberOfWorkers}",
|
||||
"--shard=${i + 1}",
|
||||
"--coverage=${coverageTmpDir.path}/",
|
||||
]));
|
||||
}
|
||||
|
||||
// Wait for isolates to terminate and clean up.
|
||||
Iterable<ProcessResult> runResults = await Future.wait(futures);
|
||||
|
||||
print("Run finished.");
|
||||
|
||||
Map<Uri, coverageMerger.CoverageInfo>? coverageData =
|
||||
coverageMerger.mergeFromDirUri(
|
||||
Uri.base.resolve(".dart_tool/package_config.json"),
|
||||
coverageTmpDir.uri,
|
||||
silent: true,
|
||||
);
|
||||
if (coverageData == null) throw "Failure in coverage.";
|
||||
|
||||
void addResult(String testName, bool pass, {String? log}) {
|
||||
results.add(jsonEncode({
|
||||
"name": "coverage/$testName",
|
||||
"configuration": options.configurationName,
|
||||
"suite": "coverage",
|
||||
"test_name": testName,
|
||||
"expected": "Pass",
|
||||
"result": pass ? "Pass" : "Fail",
|
||||
"matches": pass,
|
||||
}));
|
||||
|
||||
if (log != null) {
|
||||
logs.add(jsonEncode({
|
||||
"name": "coverage/$testName",
|
||||
"configuration": options.configurationName,
|
||||
"suite": "coverage",
|
||||
"test_name": testName,
|
||||
"result": pass ? "Pass" : "Fail",
|
||||
"log": log,
|
||||
}));
|
||||
}
|
||||
|
||||
if (options.verbose) {
|
||||
String result = pass ? "PASS" : "FAIL";
|
||||
print("${testName}: ${result}");
|
||||
}
|
||||
}
|
||||
|
||||
for (MapEntry<Uri, coverageMerger.CoverageInfo> coverageEntry
|
||||
in coverageData.entries) {
|
||||
if (coverageEntry.value.error) {
|
||||
// TODO(jensj): More info here would be good.
|
||||
addResult(coverageEntry.key.toString(), false, log: "Error");
|
||||
} else {
|
||||
int hitCount = coverageEntry.value.hitCount;
|
||||
int missCount = coverageEntry.value.missCount;
|
||||
double percent = (hitCount / (hitCount + missCount) * 100);
|
||||
if (debug) {
|
||||
print("\"${coverageEntry.key}\": $percent,");
|
||||
}
|
||||
int requireAtLeast =
|
||||
(_expect[coverageEntry.key.toString()] ?? 0.0).floor();
|
||||
bool pass = percent >= requireAtLeast;
|
||||
String? log;
|
||||
if (!pass) {
|
||||
log = "${coverageEntry.value.visualization}\n\n"
|
||||
"Expected at least $requireAtLeast%, got $percent% "
|
||||
"($hitCount hits and $missCount misses).";
|
||||
}
|
||||
addResult(coverageEntry.key.toString(), pass, log: log);
|
||||
}
|
||||
}
|
||||
|
||||
// Write results.json and logs.json.
|
||||
Uri resultJsonUri = options.outputDirectory.resolve("results.json");
|
||||
Uri logsJsonUri = options.outputDirectory.resolve("logs.json");
|
||||
await writeLinesToFile(resultJsonUri, results);
|
||||
await writeLinesToFile(logsJsonUri, logs);
|
||||
print("Log files written to ${resultJsonUri.toFilePath()} and"
|
||||
" ${logsJsonUri.toFilePath()}");
|
||||
print("Entire run took ${totalRuntime.elapsed}.");
|
||||
// Return with exit code 1 if at least one suite timed out.
|
||||
bool timedOutOrCrashed = runResults.any((p) => p.exitCode != 0);
|
||||
if (timedOutOrCrashed) {
|
||||
throw "Crashed or timed out. Check stdout for more details.";
|
||||
} else {
|
||||
exitCode = 0;
|
||||
}
|
||||
}
|
||||
|
||||
int getDefaultThreads() {
|
||||
int numberOfWorkers = 1;
|
||||
if (Platform.numberOfProcessors > 2) {
|
||||
numberOfWorkers = Platform.numberOfProcessors - 1;
|
||||
}
|
||||
if (numberOfWorkers > 5) numberOfWorkers = 5;
|
||||
return numberOfWorkers;
|
||||
}
|
||||
|
||||
Future<void> writeLinesToFile(Uri uri, List<String> lines) async {
|
||||
await File.fromUri(uri).writeAsString(lines.map((line) => "$line\n").join());
|
||||
}
|
||||
|
||||
class Options {
|
||||
final String? configurationName;
|
||||
final bool verbose;
|
||||
final bool debug;
|
||||
final Uri outputDirectory;
|
||||
final int numberOfWorkers;
|
||||
|
||||
Options(
|
||||
this.configurationName,
|
||||
this.verbose,
|
||||
this.debug,
|
||||
this.outputDirectory, {
|
||||
required this.numberOfWorkers,
|
||||
});
|
||||
|
||||
static Options parse(List<String> args) {
|
||||
var parser = new ArgParser()
|
||||
..addOption("named-configuration",
|
||||
abbr: "n",
|
||||
help: "configuration name to use for emitting json result files")
|
||||
..addOption("output-directory",
|
||||
help: "directory to which results.json and logs.json are written")
|
||||
..addFlag("verbose",
|
||||
abbr: "v", help: "print additional information", defaultsTo: false)
|
||||
..addFlag("debug", help: "debug mode", defaultsTo: false)
|
||||
..addOption("tasks",
|
||||
abbr: "j",
|
||||
help: "The number of parallel tasks to run.",
|
||||
defaultsTo: "${getDefaultThreads()}")
|
||||
// These are not used but are here for compatibility with the test system.
|
||||
..addOption("shards", help: "(Ignored) Number of shards", defaultsTo: "1")
|
||||
..addOption("shard",
|
||||
help: "(Ignored) Which shard to run", defaultsTo: "1");
|
||||
ArgResults parsedOptions = parser.parse(args);
|
||||
String outputPath = parsedOptions["output-directory"] ?? ".";
|
||||
Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath));
|
||||
|
||||
bool verbose = parsedOptions["verbose"];
|
||||
bool debug = parsedOptions["debug"];
|
||||
|
||||
String tasksString = parsedOptions["tasks"];
|
||||
int? tasks = int.tryParse(tasksString);
|
||||
if (tasks == null || tasks < 1) {
|
||||
throw "--tasks (-j) has to be an integer >= 1";
|
||||
}
|
||||
|
||||
if (verbose) {
|
||||
print("NOTE: Created with options\n "
|
||||
"named config = ${parsedOptions["named-configuration"]},\n "
|
||||
"verbose = ${verbose},\n "
|
||||
"debug = ${debug},\n "
|
||||
"${outputDirectory},\n "
|
||||
"numberOfWorkers: ${tasks}");
|
||||
}
|
||||
|
||||
return Options(
|
||||
parsedOptions["named-configuration"],
|
||||
verbose,
|
||||
debug,
|
||||
outputDirectory,
|
||||
numberOfWorkers: tasks,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -18,8 +18,8 @@ import 'testing/suite.dart';
|
|||
Future<void> internalMain(
|
||||
CreateContext createContext, {
|
||||
List<String> arguments = const [],
|
||||
int shards = 1,
|
||||
int shard = 0,
|
||||
int? shards,
|
||||
int? shard,
|
||||
required String displayName,
|
||||
String? configurationPath,
|
||||
}) async {
|
||||
|
|
@ -36,6 +36,13 @@ Future<void> internalMain(
|
|||
coverageUri = Uri.base
|
||||
.resolveUri(Uri.file(argument.substring("--coverage=".length)));
|
||||
trimmed = true;
|
||||
} else if (argument.startsWith("--shards=")) {
|
||||
shards = int.parse(argument.substring("--shards=".length));
|
||||
trimmed = true;
|
||||
} else if (argument.startsWith("--shard=")) {
|
||||
// Have this 1-indexed when given as an input.
|
||||
shard = int.parse(argument.substring("--shard=".length)) - 1;
|
||||
trimmed = true;
|
||||
}
|
||||
|
||||
if (trimmed && argumentsTrimmed == null) {
|
||||
|
|
@ -48,6 +55,9 @@ Future<void> internalMain(
|
|||
arguments = argumentsTrimmed;
|
||||
}
|
||||
|
||||
shards ??= 1;
|
||||
shard ??= 0;
|
||||
|
||||
await runMe(
|
||||
arguments,
|
||||
createContext,
|
||||
|
|
@ -57,7 +67,8 @@ Future<void> internalMain(
|
|||
logger: logger,
|
||||
);
|
||||
if (coverageUri != null) {
|
||||
File f = new File.fromUri(coverageUri.resolve("$displayName.coverage"));
|
||||
File f = new File.fromUri(
|
||||
coverageUri.resolve("$displayName.$shard.$shards.coverage"));
|
||||
// Suites generally takes a while to run --- so setting force compile to
|
||||
// true shouldn't be a big issue. It seems to add something like a second
|
||||
// to the collection time.
|
||||
|
|
|
|||
|
|
@ -112,5 +112,8 @@ Future<void> main() async {
|
|||
// and const classes etc that shouldn't (necessarily) be compiled but is
|
||||
// potentially covered in other ways.
|
||||
coverageMerger.mergeFromDirUri(
|
||||
repoDirUri.resolve(".dart_tool/package_config.json"), coverageTmpDir.uri);
|
||||
repoDirUri.resolve(".dart_tool/package_config.json"),
|
||||
coverageTmpDir.uri,
|
||||
silent: false,
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ bat
|
|||
bazbaz
|
||||
bbb
|
||||
bc
|
||||
bd
|
||||
bench
|
||||
benchmarker
|
||||
benchmarkers
|
||||
|
|
@ -179,6 +180,7 @@ consts
|
|||
contract
|
||||
conversions
|
||||
coo
|
||||
cores
|
||||
corge
|
||||
corners
|
||||
costing
|
||||
|
|
@ -790,6 +792,7 @@ theoretically
|
|||
thereby
|
||||
thereof
|
||||
thread
|
||||
threads
|
||||
ticks
|
||||
tile
|
||||
timed
|
||||
|
|
@ -854,6 +857,7 @@ val
|
|||
verbatim
|
||||
versioned
|
||||
vf
|
||||
visualization
|
||||
vp
|
||||
vt
|
||||
vte
|
||||
|
|
@ -869,6 +873,7 @@ whiskers
|
|||
wildcards
|
||||
wins
|
||||
wording
|
||||
workers
|
||||
workflow
|
||||
worlds
|
||||
wrongly
|
||||
|
|
|
|||
|
|
@ -42,19 +42,28 @@ void main(List<String> arguments) {
|
|||
}
|
||||
|
||||
Stopwatch stopwatch = new Stopwatch()..start();
|
||||
mergeFromDirUri(packagesUri, coverageUri);
|
||||
mergeFromDirUri(packagesUri, coverageUri, silent: false);
|
||||
print("Done in ${stopwatch.elapsed}");
|
||||
}
|
||||
|
||||
void mergeFromDirUri(Uri packagesUri, Uri coverageUri) {
|
||||
Map<Uri, CoverageInfo>? mergeFromDirUri(
|
||||
Uri packagesUri,
|
||||
Uri coverageUri, {
|
||||
required bool silent,
|
||||
}) {
|
||||
void output(Object? object) {
|
||||
if (silent) return;
|
||||
print(object);
|
||||
}
|
||||
|
||||
PackageConfig packageConfig;
|
||||
try {
|
||||
packageConfig = PackageConfig.parseBytes(
|
||||
File.fromUri(packagesUri).readAsBytesSync(), packagesUri);
|
||||
} catch (e) {
|
||||
// When will we want to catch this?
|
||||
print("Error trying to read package config: $e");
|
||||
return;
|
||||
output("Error trying to read package config: $e");
|
||||
return null;
|
||||
}
|
||||
|
||||
// TODO(jensj): We should allow for comments to "excuse" something from not
|
||||
|
|
@ -84,13 +93,13 @@ void mergeFromDirUri(Uri packagesUri, Uri coverageUri) {
|
|||
if (entry is! File) continue;
|
||||
try {
|
||||
Coverage coverage = Coverage.loadFromFile(entry);
|
||||
print("Loaded $entry as coverage file.");
|
||||
output("Loaded $entry as coverage file.");
|
||||
_mergeCoverageInto(coverage, misses, hits);
|
||||
} catch (e) {
|
||||
print("Couldn't load $entry as coverage file.");
|
||||
output("Couldn't load $entry as coverage file.");
|
||||
}
|
||||
}
|
||||
print("");
|
||||
output("");
|
||||
|
||||
Set<Uri> knownUris = {};
|
||||
knownUris.addAll(hits.keys);
|
||||
|
|
@ -102,6 +111,8 @@ void mergeFromDirUri(Uri packagesUri, Uri coverageUri) {
|
|||
int missesTotal = 0;
|
||||
int errorsCount = 0;
|
||||
|
||||
Map<Uri, CoverageInfo> result = {};
|
||||
|
||||
for (Uri uri in knownUris.toList()
|
||||
..sort(((a, b) => a.toString().compareTo(b.toString())))) {
|
||||
// Don't care about coverage for testing stuff.
|
||||
|
|
@ -112,9 +123,10 @@ void mergeFromDirUri(Uri packagesUri, Uri coverageUri) {
|
|||
List<int> hitsSorted =
|
||||
hit == null ? const [] : (hit._data.keys.toList()..sort());
|
||||
|
||||
ProcessInfo processInfo = process(
|
||||
packageConfig, uri, miss ?? const {}, hitsSorted,
|
||||
visualize: true);
|
||||
CoverageInfo processInfo =
|
||||
process(packageConfig, uri, miss ?? const {}, hitsSorted);
|
||||
output(processInfo.visualization);
|
||||
result[uri] = processInfo;
|
||||
filesCount++;
|
||||
if (processInfo.error) {
|
||||
errorsCount++;
|
||||
|
|
@ -126,49 +138,50 @@ void mergeFromDirUri(Uri packagesUri, Uri coverageUri) {
|
|||
missesTotal += processInfo.missCount;
|
||||
}
|
||||
|
||||
print("");
|
||||
output("");
|
||||
}
|
||||
|
||||
print("Processed $filesCount files with $errorsCount error(s) and "
|
||||
output("Processed $filesCount files with $errorsCount error(s) and "
|
||||
"$allCoveredCount files being covered 100%.");
|
||||
int percentHit = (hitsTotal * 100) ~/ (hitsTotal + missesTotal);
|
||||
print("All-in-all $hitsTotal hits and $missesTotal misses ($percentHit%).");
|
||||
output("All-in-all $hitsTotal hits and $missesTotal misses ($percentHit%).");
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
class ProcessInfo {
|
||||
class CoverageInfo {
|
||||
final bool error;
|
||||
final bool allCovered;
|
||||
final int missCount;
|
||||
final int hitCount;
|
||||
final String visualization;
|
||||
|
||||
ProcessInfo.error()
|
||||
CoverageInfo.error(this.visualization)
|
||||
: error = true,
|
||||
allCovered = false,
|
||||
missCount = 0,
|
||||
hitCount = 0;
|
||||
|
||||
ProcessInfo(
|
||||
CoverageInfo(
|
||||
{required this.allCovered,
|
||||
required this.missCount,
|
||||
required this.hitCount})
|
||||
required this.hitCount,
|
||||
required this.visualization})
|
||||
: error = false;
|
||||
}
|
||||
|
||||
ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
||||
Set<int> untrimmedMisses, List<int> hitsSorted,
|
||||
{required bool visualize}) {
|
||||
CoverageInfo process(PackageConfig packageConfig, Uri uri,
|
||||
Set<int> untrimmedMisses, List<int> hitsSorted) {
|
||||
Uri? fileUri = packageConfig.resolve(uri);
|
||||
if (fileUri == null) {
|
||||
print("Couldn't find file uri for $uri");
|
||||
return new ProcessInfo.error();
|
||||
return new CoverageInfo.error("Couldn't find file uri for $uri");
|
||||
}
|
||||
File f = new File.fromUri(fileUri);
|
||||
Uint8List rawBytes;
|
||||
try {
|
||||
rawBytes = f.readAsBytesSync();
|
||||
} catch (e) {
|
||||
print("Error reading file $f");
|
||||
return new ProcessInfo.error();
|
||||
return new CoverageInfo.error("Error reading file $f");
|
||||
}
|
||||
|
||||
List<int> lineStarts = [];
|
||||
|
|
@ -190,17 +203,20 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
// TODO(jensj): Should some comment throw/report and error if covered?
|
||||
// E.g. "we expect this to be dead code, if it isn't we want to know."
|
||||
|
||||
StringBuffer visualization = new StringBuffer();
|
||||
|
||||
IntervalList ignoredIntervals =
|
||||
astIndexer.ignoredStartEnd.buildIntervalList();
|
||||
var (:bool allCovered, :Set<int> trimmedMisses) =
|
||||
_trimIgnoredAndPrintPercentages(
|
||||
ignoredIntervals, untrimmedMisses, hitsSorted, uri);
|
||||
visualization, ignoredIntervals, untrimmedMisses, hitsSorted, uri);
|
||||
|
||||
if (!visualize || allCovered) {
|
||||
return new ProcessInfo(
|
||||
if (allCovered) {
|
||||
return new CoverageInfo(
|
||||
allCovered: allCovered,
|
||||
missCount: trimmedMisses.length,
|
||||
hitCount: hitsSorted.length);
|
||||
hitCount: hitsSorted.length,
|
||||
visualization: visualization.toString());
|
||||
}
|
||||
|
||||
CompilationUnitBegin unitBegin = ast.children!.first as CompilationUnitBegin;
|
||||
|
|
@ -222,9 +238,9 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
String? name = astIndexer.nameOfEntitySpanning(lastOffset);
|
||||
String pointer = new String.fromCharCodes(indentation!);
|
||||
if (name != null) {
|
||||
print("$uri:$lastLine:\nIn '$name':\n$line\n$pointer");
|
||||
visualization.writeln("$uri:$lastLine:\nIn '$name':\n$line\n$pointer");
|
||||
} else {
|
||||
print("$uri:$lastLine:\n$line\n$pointer");
|
||||
visualization.writeln("$uri:$lastLine:\n$line\n$pointer");
|
||||
}
|
||||
line = null;
|
||||
indentation = null;
|
||||
|
|
@ -269,10 +285,12 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
String? name = astIndexer.nameOfEntitySpanning(offset);
|
||||
|
||||
if (name != null) {
|
||||
print("$uri:${location.line}: No coverage for '$name'.\n$line\n");
|
||||
visualization.writeln(
|
||||
"$uri:${location.line}: No coverage for '$name'.\n$line\n");
|
||||
// TODO(jensj): Squiggly line under the identifier of the entity?
|
||||
} else {
|
||||
print("$uri:${location.line}: No coverage for entity.\n$line\n");
|
||||
visualization.writeln(
|
||||
"$uri:${location.line}: No coverage for entity.\n$line\n");
|
||||
}
|
||||
|
||||
// Skip the rest of the miss points inside the entity.
|
||||
|
|
@ -305,20 +323,23 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
}
|
||||
lastOffset = offset;
|
||||
} catch (e) {
|
||||
print("Error on offset $offset --- $location: $e");
|
||||
print("Maybe the coverage data is not up to date with the source?");
|
||||
rethrow;
|
||||
visualization.writeln("Error on offset $offset --- $location: $e");
|
||||
visualization.writeln(
|
||||
"Maybe the coverage data is not up to date with the source?");
|
||||
return new CoverageInfo.error(visualization.toString());
|
||||
}
|
||||
}
|
||||
printFinishedLine();
|
||||
|
||||
return new ProcessInfo(
|
||||
return new CoverageInfo(
|
||||
allCovered: allCovered,
|
||||
missCount: trimmedMisses.length,
|
||||
hitCount: hitsSorted.length);
|
||||
hitCount: hitsSorted.length,
|
||||
visualization: visualization.toString());
|
||||
}
|
||||
|
||||
({bool allCovered, Set<int> trimmedMisses}) _trimIgnoredAndPrintPercentages(
|
||||
StringBuffer visualization,
|
||||
IntervalList ignoredIntervals,
|
||||
Set<int> untrimmedMisses,
|
||||
List<int> hitsSorted,
|
||||
|
|
@ -327,7 +348,7 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
int hitCount = hitsSorted.length;
|
||||
Set<int> trimmedMisses;
|
||||
if (hitCount + missCount == 0) {
|
||||
print("$uri");
|
||||
visualization.writeln("$uri");
|
||||
return (allCovered: true, trimmedMisses: untrimmedMisses);
|
||||
} else {
|
||||
if (!ignoredIntervals.isEmpty) {
|
||||
|
|
@ -345,11 +366,12 @@ ProcessInfo process(PackageConfig packageConfig, Uri uri,
|
|||
}
|
||||
|
||||
if (missCount > 0) {
|
||||
print("$uri: ${(hitCount / (hitCount + missCount) * 100).round()}% "
|
||||
visualization.writeln(
|
||||
"$uri: ${(hitCount / (hitCount + missCount) * 100).round()}% "
|
||||
"($missCount misses)");
|
||||
return (allCovered: false, trimmedMisses: trimmedMisses);
|
||||
} else {
|
||||
print("$uri: 100% (OK)");
|
||||
visualization.writeln("$uri: 100% (OK)");
|
||||
return (allCovered: true, trimmedMisses: trimmedMisses);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -198,7 +198,7 @@ class AstIndexer extends IgnoreSomeForCompatibilityAstVisitor {
|
|||
void containerMethod(
|
||||
BeginAndEndTokenParserAstNode node, String nameIdentifier) {
|
||||
positionStartEndIndex.add(node.beginToken.charOffset);
|
||||
positionStartEndIndex.add(node.beginToken.charEnd);
|
||||
positionStartEndIndex.add(node.endToken.charEnd);
|
||||
// TODO(jensj): Setters.
|
||||
String name = "$currentContainerName.$nameIdentifier";
|
||||
nameIndex[name] = positionNodeIndex.length;
|
||||
|
|
|
|||
|
|
@ -673,6 +673,18 @@
|
|||
"fileset": "front-end",
|
||||
"shards": 3
|
||||
},
|
||||
{
|
||||
"name": "front_end coverage",
|
||||
"script": "out/ReleaseX64/dart-sdk/bin/dart",
|
||||
"testRunner": true,
|
||||
"arguments": [
|
||||
"--enable-asserts",
|
||||
"pkg/front_end/test/coverage_suite.dart",
|
||||
"-ncfe-unittest-asserts-${mode}-${system}"
|
||||
],
|
||||
"fileset": "front-end",
|
||||
"shards": 1
|
||||
},
|
||||
{
|
||||
"name": "front_end unit tests (with git)",
|
||||
"arguments": [
|
||||
|
|
|
|||
Loading…
Reference in a new issue