[vm] Adding branch coverage RPC to source report

TEST=Unit tests and an integration test
Bug: https://github.com/dart-lang/coverage/issues/141
Change-Id: I84958091dc6f9753f5e9446bb3517a8099019981
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/222541
Reviewed-by: Ben Konyi <bkonyi@google.com>
Commit-Queue: Liam Appelbe <liama@google.com>
This commit is contained in:
Liam Appelbe 2022-01-12 19:12:04 +00:00 committed by Commit Bot
parent d1135bbbd5
commit 9d79a890f3
11 changed files with 448 additions and 23 deletions

View file

@ -223,7 +223,8 @@ class RecursiveContinuationRewriter extends RemovingTransformer {
resultType: elementType)
..fileOffset = stmt.bodyOffset);
final Block body = Block([variable, stmt.body]);
final Block body = Block([variable, stmt.body])
..fileOffset = stmt.bodyOffset;
return transform(
Block([syncForIterator, ForStatement([], condition, [], body)]));

View file

@ -3,6 +3,8 @@
## 8.2.0-dev
- Update to version `3.56` of the spec.
- Added optional `line` and `column` properties to `SourceLocation`.
- Added a new `SourceReportKind`, `BranchCoverage`, which reports branch level
coverage information.
## 8.1.0
- Update to version `3.55` of the spec.

View file

@ -230,6 +230,7 @@ String assertFrameKind(String obj) {
}
String assertSourceReportKind(String obj) {
if (obj == "BranchCoverage") return obj;
if (obj == "Coverage") return obj;
if (obj == "PossibleBreakpoints") return obj;
throw "invalid SourceReportKind: $obj";

View file

@ -2776,6 +2776,9 @@ class SourceReportKind {
/// Used to request a list of token positions of possible breakpoints.
static const String kPossibleBreakpoints = 'PossibleBreakpoints';
/// Used to request branch coverage information.
static const String kBranchCoverage = 'BranchCoverage';
}
/// An `ExceptionPauseMode` indicates how the isolate pauses when an exception
@ -7594,6 +7597,11 @@ class SourceReportRange {
@optional
List<int>? possibleBreakpoints;
/// Branch coverage information for this range. Provided only when the
/// BranchCoverage report has been requested and the range has been compiled.
@optional
SourceReportCoverage? branchCoverage;
SourceReportRange({
required this.scriptIndex,
required this.startPos,
@ -7602,6 +7610,7 @@ class SourceReportRange {
this.error,
this.coverage,
this.possibleBreakpoints,
this.branchCoverage,
});
SourceReportRange._fromJson(Map<String, dynamic> json) {
@ -7615,6 +7624,9 @@ class SourceReportRange {
possibleBreakpoints = json['possibleBreakpoints'] == null
? null
: List<int>.from(json['possibleBreakpoints']);
branchCoverage = createServiceObject(
json['branchCoverage'], const ['SourceReportCoverage'])
as SourceReportCoverage?;
}
Map<String, dynamic> toJson() {
@ -7629,6 +7641,7 @@ class SourceReportRange {
_setIfNotNull(json, 'coverage', coverage?.toJson());
_setIfNotNull(json, 'possibleBreakpoints',
possibleBreakpoints?.map((f) => f).toList());
_setIfNotNull(json, 'branchCoverage', branchCoverage?.toJson());
return json;
}

View file

@ -0,0 +1,140 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:developer';
import 'package:test/test.dart';
import 'package:vm_service/vm_service.dart';
import 'common/service_test_common.dart';
import 'common/test_helper.dart';
int ifTest(x) {
if (x > 0) {
if (x > 10) {
return 10;
} else {
return 1;
}
} else {
return 0;
}
}
void testFunction() {
debugger();
ifTest(1);
debugger();
}
bool allRangesCompiled(coverage) {
for (int i = 0; i < coverage['ranges'].length; i++) {
if (!coverage['ranges'][i]['compiled']) {
return false;
}
}
return true;
}
IsolateTest coverageTest(
Map<String, dynamic> expectedRange, {
required bool reportLines,
}) {
return (VmService service, IsolateRef isolateRef) async {
final isolateId = isolateRef.id!;
final isolate = await service.getIsolate(isolateId);
final stack = await service.getStack(isolateId);
// Make sure we are in the right place.
expect(stack.frames!.length, greaterThanOrEqualTo(1));
expect(stack.frames![0].function!.name, 'testFunction');
final root =
await service.getObject(isolateId, isolate.rootLib!.id!) as Library;
final funcRef = root.functions!.singleWhere((f) => f.name == 'ifTest');
final func = await service.getObject(isolateId, funcRef.id!) as Func;
final location = func.location!;
final report = await service.getSourceReport(
isolateId,
[SourceReportKind.kBranchCoverage],
scriptId: location.script!.id,
tokenPos: location.tokenPos,
endTokenPos: location.endTokenPos,
forceCompile: true,
reportLines: reportLines,
);
expect(report.ranges!.length, 1);
expect(report.ranges![0].toJson(), expectedRange);
expect(report.scripts!.length, 1);
expect(
report.scripts![0].uri,
endsWith('branch_coverage_test.dart'),
);
};
}
var tests = <IsolateTest>[
hasStoppedAtBreakpoint,
coverageTest(
{
'scriptIndex': 0,
'startPos': 397,
'endPos': 527,
'compiled': true,
'branchCoverage': {
'hits': [],
'misses': [397, 426, 444, 474, 507]
}
},
reportLines: false,
),
coverageTest(
{
'scriptIndex': 0,
'startPos': 397,
'endPos': 527,
'compiled': true,
'branchCoverage': {
'hits': [],
'misses': [11, 12, 13, 15, 18]
}
},
reportLines: true,
),
resumeIsolate,
hasStoppedAtBreakpoint,
coverageTest(
{
'scriptIndex': 0,
'startPos': 397,
'endPos': 527,
'compiled': true,
'branchCoverage': {
'hits': [397, 426, 474],
'misses': [444, 507]
}
},
reportLines: false,
),
coverageTest(
{
'scriptIndex': 0,
'startPos': 397,
'endPos': 527,
'compiled': true,
'branchCoverage': {
'hits': [11, 12, 15],
'misses': [13, 18]
}
},
reportLines: true,
),
];
main([args = const <String>[]]) => runIsolateTests(
args,
tests,
'branch_coverage_test.dart',
testeeConcurrent: testFunction,
extraArgs: ['--branch-coverage'],
);

View file

@ -489,6 +489,11 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
EnableAssertsBit::update(value, isolate_group_flags_);
}
void set_branch_coverage(bool value) {
isolate_group_flags_ =
BranchCoverageBit::update(value, isolate_group_flags_);
}
#if !defined(PRODUCT)
#if !defined(DART_PRECOMPILED_RUNTIME)
bool HasAttemptedReload() const {

View file

@ -3464,11 +3464,9 @@ static void GetPorts(Thread* thread, JSONStream* js) {
#if !defined(DART_PRECOMPILED_RUNTIME)
static const char* const report_enum_names[] = {
SourceReport::kCallSitesStr,
SourceReport::kCoverageStr,
SourceReport::kPossibleBreakpointsStr,
SourceReport::kProfileStr,
NULL,
SourceReport::kCallSitesStr, SourceReport::kCoverageStr,
SourceReport::kPossibleBreakpointsStr, SourceReport::kProfileStr,
SourceReport::kBranchCoverageStr, NULL,
};
#endif
@ -3507,6 +3505,8 @@ static void GetSourceReport(Thread* thread, JSONStream* js) {
report_set |= SourceReport::kPossibleBreakpoints;
} else if (strcmp(*riter, SourceReport::kProfileStr) == 0) {
report_set |= SourceReport::kProfile;
} else if (strcmp(*riter, SourceReport::kBranchCoverageStr) == 0) {
report_set |= SourceReport::kBranchCoverage;
}
riter++;
}

View file

@ -3942,7 +3942,10 @@ enum SourceReportKind {
Coverage,
// Used to request a list of token positions of possible breakpoints.
PossibleBreakpoints
PossibleBreakpoints,
// Used to request branch coverage information.
BranchCoverage
}
```
@ -3977,6 +3980,11 @@ class SourceReportRange {
// enabled). Provided only when the when the PossibleBreakpoint report has
// been requested and the range has been compiled.
int[] possibleBreakpoints [optional];
// Branch coverage information for this range. Provided only when the
// BranchCoverage report has been requested and the range has been
// compiled.
SourceReportCoverage branchCoverage [optional];
}
```
@ -4327,6 +4335,6 @@ version | comments
3.53 | Added `setIsolatePauseMode` RPC.
3.54 | Added `CpuSamplesEvent`, updated `cpuSamples` property on `Event` to have type `CpuSamplesEvent`.
3.55 | Added `streamCpuSamplesWithUserTag` RPC.
3.56 | Added optional `line` and `column` properties to `SourceLocation`.
3.56 | Added optional `line` and `column` properties to `SourceLocation`. Added a new `SourceReportKind`, `BranchCoverage`, which reports branch level coverage information.
[discuss-list]: https://groups.google.com/a/dartlang.org/forum/#!forum/observatory-discuss

View file

@ -21,6 +21,7 @@ const char* SourceReport::kCallSitesStr = "_CallSites";
const char* SourceReport::kCoverageStr = "Coverage";
const char* SourceReport::kPossibleBreakpointsStr = "PossibleBreakpoints";
const char* SourceReport::kProfileStr = "_Profile";
const char* SourceReport::kBranchCoverageStr = "BranchCoverage";
SourceReport::SourceReport(intptr_t report_set,
CompileMode compile_mode,
@ -285,7 +286,8 @@ bool SourceReport::ShouldCoverageSkipCallSite(const ICData* ic_data) {
void SourceReport::PrintCoverageData(JSONObject* jsobj,
const Function& function,
const Code& code) {
const Code& code,
bool report_branch_coverage) {
ASSERT(!code.IsNull());
const TokenPosition& begin_pos = function.token_pos();
const TokenPosition& end_pos = function.end_token_pos();
@ -329,16 +331,18 @@ void SourceReport::PrintCoverageData(JSONObject* jsobj,
}
};
PcDescriptors::Iterator iter(
descriptors,
UntaggedPcDescriptors::kIcCall | UntaggedPcDescriptors::kUnoptStaticCall);
while (iter.MoveNext()) {
HANDLESCOPE(thread());
ASSERT(iter.DeoptId() < ic_data_array->length());
const ICData* ic_data = (*ic_data_array)[iter.DeoptId()];
if (!ShouldCoverageSkipCallSite(ic_data)) {
const TokenPosition& token_pos = iter.TokenPos();
update_coverage(token_pos, ic_data->AggregateCount() > 0);
if (!report_branch_coverage) {
PcDescriptors::Iterator iter(descriptors,
UntaggedPcDescriptors::kIcCall |
UntaggedPcDescriptors::kUnoptStaticCall);
while (iter.MoveNext()) {
HANDLESCOPE(thread());
ASSERT(iter.DeoptId() < ic_data_array->length());
const ICData* ic_data = (*ic_data_array)[iter.DeoptId()];
if (!ShouldCoverageSkipCallSite(ic_data)) {
const TokenPosition& token_pos = iter.TokenPos();
update_coverage(token_pos, ic_data->AggregateCount() > 0);
}
}
}
@ -349,7 +353,7 @@ void SourceReport::PrintCoverageData(JSONObject* jsobj,
bool is_branch_coverage;
const TokenPosition token_pos = TokenPosition::DecodeCoveragePosition(
Smi::Value(Smi::RawCast(coverage_array.At(i))), &is_branch_coverage);
if (!is_branch_coverage) {
if (is_branch_coverage == report_branch_coverage) {
const bool was_executed =
Smi::Value(Smi::RawCast(coverage_array.At(i + 1))) != 0;
update_coverage(token_pos, was_executed);
@ -357,7 +361,7 @@ void SourceReport::PrintCoverageData(JSONObject* jsobj,
}
}
JSONObject cov(jsobj, "coverage");
JSONObject cov(jsobj, report_branch_coverage ? "branchCoverage" : "coverage");
{
JSONArray hits(&cov, "hits");
TokenPosition pos = begin_pos;
@ -532,7 +536,10 @@ void SourceReport::VisitFunction(JSONArray* jsarr, const Function& func) {
PrintCallSitesData(&range, func, code);
}
if (IsReportRequested(kCoverage)) {
PrintCoverageData(&range, func, code);
PrintCoverageData(&range, func, code, /* report_branch_coverage */ false);
}
if (IsReportRequested(kBranchCoverage)) {
PrintCoverageData(&range, func, code, /* report_branch_coverage */ true);
}
if (IsReportRequested(kPossibleBreakpoints)) {
PrintPossibleBreakpointsData(&range, func, code);

View file

@ -27,12 +27,14 @@ class SourceReport {
kCoverage = 0x2,
kPossibleBreakpoints = 0x4,
kProfile = 0x8,
kBranchCoverage = 0x10,
};
static const char* kCallSitesStr;
static const char* kCoverageStr;
static const char* kPossibleBreakpointsStr;
static const char* kProfileStr;
static const char* kBranchCoverageStr;
enum CompileMode { kNoCompile, kForceCompile };
@ -77,7 +79,8 @@ class SourceReport {
const Code& code);
void PrintCoverageData(JSONObject* jsobj,
const Function& func,
const Code& code);
const Code& code,
bool report_branch_coverage);
void PrintPossibleBreakpointsData(JSONObject* jsobj,
const Function& func,
const Code& code);

View file

@ -1127,6 +1127,251 @@ main() {
buffer);
}
ISOLATE_UNIT_TEST_CASE(SourceReport_BranchCoverage_if) {
// WARNING: This MUST be big enough for the serialised JSON string.
const int kBufferSize = 1024;
char buffer[kBufferSize];
const char* kScript = R"(
int ifTest(int x) {
if (x > 0) {
if (x > 10) {
return 10;
} else {
return 1;
}
} else {
return 0;
}
}
main() {
ifTest(1);
}
)";
Library& lib = Library::Handle();
const bool old_branch_coverage = IsolateGroup::Current()->branch_coverage();
IsolateGroup::Current()->set_branch_coverage(true);
lib ^= ExecuteScript(kScript);
IsolateGroup::Current()->set_branch_coverage(old_branch_coverage);
ASSERT(!lib.IsNull());
const Script& script =
Script::Handle(lib.LookupScript(String::Handle(String::New("test-lib"))));
SourceReport report(SourceReport::kBranchCoverage);
JSONStream js;
report.PrintJSON(&js, script);
const char* json_str = js.ToCString();
ASSERT(strlen(json_str) < kBufferSize);
ElideJSONSubstring("classes", json_str, buffer);
ElideJSONSubstring("libraries", buffer, buffer);
EXPECT_STREQ(
"{\"type\":\"SourceReport\",\"ranges\":["
// In ifTest, the outer true case is hit, the inner true case is missed,
// the inner false case is hit, and the outer false case is missed.
"{\"scriptIndex\":0,\"startPos\":1,\"endPos\":135,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[1,34,82],\"misses\":[52,115]}},"
// Main is hit.
"{\"scriptIndex\":0,\"startPos\":138,\"endPos\":160,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[138],\"misses\":[]}}],"
// Only one script in the script table.
"\"scripts\":[{\"type\":\"@Script\",\"fixedId\":true,\"id\":\"\","
"\"uri\":\"file:\\/\\/\\/test-lib\",\"_kind\":\"kernel\"}]}",
buffer);
}
ISOLATE_UNIT_TEST_CASE(SourceReport_BranchCoverage_loops) {
// WARNING: This MUST be big enough for the serialised JSON string.
const int kBufferSize = 1024;
char buffer[kBufferSize];
const char* kScript = R"(
int loopTest() {
var x = 0;
while (x < 10) {
++x;
}
do {
++x;
} while (false);
for (int i = 0; i < 10; ++i) {
++x;
}
for (final i in [1, 2, 3]) {
++x;
}
return x;
}
main() {
loopTest();
}
)";
Library& lib = Library::Handle();
const bool old_branch_coverage = IsolateGroup::Current()->branch_coverage();
IsolateGroup::Current()->set_branch_coverage(true);
lib ^= ExecuteScript(kScript);
IsolateGroup::Current()->set_branch_coverage(old_branch_coverage);
ASSERT(!lib.IsNull());
const Script& script =
Script::Handle(lib.LookupScript(String::Handle(String::New("test-lib"))));
SourceReport report(SourceReport::kBranchCoverage);
JSONStream js;
report.PrintJSON(&js, script);
const char* json_str = js.ToCString();
ASSERT(strlen(json_str) < kBufferSize);
ElideJSONSubstring("classes", json_str, buffer);
ElideJSONSubstring("libraries", buffer, buffer);
EXPECT_STREQ(
"{\"type\":\"SourceReport\",\"ranges\":["
// In loopTest, the while loop, do-while loop, for loop, and for-in loop
// are all hit.
"{\"scriptIndex\":0,\"startPos\":1,\"endPos\":205,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[1,49,70,132,177],\"misses\":[]}},"
// Main is hit.
"{\"scriptIndex\":0,\"startPos\":208,\"endPos\":231,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[208],\"misses\":[]}}],"
// Only one script in the script table.
"\"scripts\":[{\"type\":\"@Script\",\"fixedId\":true,\"id\":\"\","
"\"uri\":\"file:\\/\\/\\/test-lib\",\"_kind\":\"kernel\"}]}",
buffer);
}
ISOLATE_UNIT_TEST_CASE(SourceReport_BranchCoverage_switch) {
// WARNING: This MUST be big enough for the serialised JSON string.
const int kBufferSize = 1024;
char buffer[kBufferSize];
const char* kScript = R"(
int switchTest(int x) {
switch (x) {
case 0:
return 10;
case 1:
return 20;
default:
return 30;
}
}
main() {
switchTest(1);
}
)";
Library& lib = Library::Handle();
const bool old_branch_coverage = IsolateGroup::Current()->branch_coverage();
IsolateGroup::Current()->set_branch_coverage(true);
lib ^= ExecuteScript(kScript);
IsolateGroup::Current()->set_branch_coverage(old_branch_coverage);
ASSERT(!lib.IsNull());
const Script& script =
Script::Handle(lib.LookupScript(String::Handle(String::New("test-lib"))));
SourceReport report(SourceReport::kBranchCoverage);
JSONStream js;
report.PrintJSON(&js, script);
const char* json_str = js.ToCString();
ASSERT(strlen(json_str) < kBufferSize);
ElideJSONSubstring("classes", json_str, buffer);
ElideJSONSubstring("libraries", buffer, buffer);
EXPECT_STREQ(
"{\"type\":\"SourceReport\",\"ranges\":["
// In switchTest, the 1 case is hit and the others are missed.
"{\"scriptIndex\":0,\"startPos\":1,\"endPos\":132,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[1,73],\"misses\":[44,102]}},"
// Main is hit.
"{\"scriptIndex\":0,\"startPos\":135,\"endPos\":161,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[135],\"misses\":[]}}],"
// Only one script in the script table.
"\"scripts\":[{\"type\":\"@Script\",\"fixedId\":true,\"id\":\"\","
"\"uri\":\"file:\\/\\/\\/test-lib\",\"_kind\":\"kernel\"}]}",
buffer);
}
ISOLATE_UNIT_TEST_CASE(SourceReport_BranchCoverage_try) {
// WARNING: This MUST be big enough for the serialised JSON string.
const int kBufferSize = 1024;
char buffer[kBufferSize];
const char* kScript = R"(
void tryTestInner() {
try {
throw "abc";
} catch (e) {
} finally {
}
try {
throw "def";
} finally {
}
}
void tryTestOuter() {
try {
tryTestInner();
} catch (e) {
}
}
main() {
tryTestOuter();
}
)";
Library& lib = Library::Handle();
const bool old_branch_coverage = IsolateGroup::Current()->branch_coverage();
IsolateGroup::Current()->set_branch_coverage(true);
lib ^= ExecuteScript(kScript);
IsolateGroup::Current()->set_branch_coverage(old_branch_coverage);
ASSERT(!lib.IsNull());
const Script& script =
Script::Handle(lib.LookupScript(String::Handle(String::New("test-lib"))));
SourceReport report(SourceReport::kBranchCoverage);
JSONStream js;
report.PrintJSON(&js, script);
const char* json_str = js.ToCString();
ASSERT(strlen(json_str) < kBufferSize);
ElideJSONSubstring("classes", json_str, buffer);
ElideJSONSubstring("libraries", buffer, buffer);
EXPECT_STREQ(
"{\"type\":\"SourceReport\",\"ranges\":["
// In tryTestInner, the try/catch/finally and the try/finally are all hit,
// and the try/finally rethrows its exception.
"{\"scriptIndex\":0,\"startPos\":1,\"endPos\":126,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[1,29,62,76,89,120],\"misses\":[]}},"
// In tryTestOuter, the exception thrown by tryTestInner causes both the
// try and the catch to be hit.
"{\"scriptIndex\":0,\"startPos\":129,\"endPos\":199,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[129,157,193],\"misses\":[]}},"
// Main is hit.
"{\"scriptIndex\":0,\"startPos\":202,\"endPos\":229,\"compiled\":true,"
"\"branchCoverage\":{\"hits\":[202],\"misses\":[]}}],"
// Only one script in the script table.
"\"scripts\":[{\"type\":\"@Script\",\"fixedId\":true,\"id\":\"\","
"\"uri\":\"file:\\/\\/\\/test-lib\",\"_kind\":\"kernel\"}]}",
buffer);
}
#endif // !PRODUCT
} // namespace dart