languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-10-08 18:24:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improve the output from the code completion metrics tool

Browse source 
This CL does three things.

First, it makes the output options more fine grained to allow us to get
the information we're after without getting information that we don't
care about at the moment.

Second, it prints the unconditional output in a more compact tabular
form so that it's easier to read.

Third, it adds support for outputing a high-level comparison of multiple
experiments to make it easier to gauage the impact of a change.

Change-Id: Iee8b1eb126de43cb08c27a1b60036090fffc0421
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/174941
Reviewed-by: Phil Quitslund <pquitslund@google.com>
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
Brian Wilkerson 2020-12-03 17:14:13 +00:00 committed by commit-bot@chromium.org
parent a3236f6622
commit 3f2e5d8a90
3 changed files with 244 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

286
pkg/analysis_server/tool/code_completion/completion_metrics.dart
View file

@ -54,14 +54,19 @@ Future<void> main(List<String> args) async {
return io.exit(1);
}
var options = CompletionMetricsOptions(
availableSuggestions: result[AVAILABLE_SUGGESTIONS],
overlay: result[OVERLAY],
printMissedCompletionDetails: result[PRINT_MISSED_COMPLETION_DETAILS],
printMissedCompletionSummary: result[PRINT_MISSED_COMPLETION_SUMMARY],
printMissingInformation: result[PRINT_MISSING_INFORMATION],
printMrrByLocation: result[PRINT_MRR_BY_LOCATION],
printSlowestResults: result[PRINT_SLOWEST_RESULTS],
printWorstResults: result[PRINT_WORST_RESULTS]);
var root = result.rest[0];
print('Analyzing root: "$root"');
var stopwatch = Stopwatch()..start();
var code = await CompletionMetricsComputer(root,
availableSuggestions: result[AVAILABLE_SUGGESTIONS],
overlay: result[OVERLAY],
verbose: result[VERBOSE])
.compute();
var code = await CompletionMetricsComputer(root, options).compute();
stopwatch.stop();
var duration = Duration(milliseconds: stopwatch.elapsedMilliseconds);
@ -86,8 +91,34 @@ const String OVERLAY_REMOVE_REST_OF_FILE = 'remove-rest-of-file';
/// completion offset should be removed.
const String OVERLAY_REMOVE_TOKEN = 'remove-token';
/// A flag that causes additional output to be produced.
const String VERBOSE = 'verbose';
/// A flag that causes detailed information to be printed every time a
/// completion request fails to produce a suggestions matching the expected
/// suggestion.
const String PRINT_MISSED_COMPLETION_DETAILS =
'print-missed-completion-details';
/// A flag that causes summary information to be printed about the times that a
/// completion request failed to produce a suggestions matching the expected
/// suggestion.
const String PRINT_MISSED_COMPLETION_SUMMARY =
'print-missed-completion-summary';
/// A flag that causes information to be printed about places where no
/// completion location was computed and about information that's missing in the
/// completion tables.
const String PRINT_MISSING_INFORMATION = 'print-missing-information';
/// A flag that causes information to be printed about the mrr score achieved at
/// each completion location.
const String PRINT_MRR_BY_LOCATION = 'print-mrr-by-location';
/// A flag that causes information to be printed about the completion requests
/// that were the slowest to return suggestions.
const String PRINT_SLOWEST_RESULTS = 'print-slowest-results';
/// A flag that causes information to be printed about the completion requests
/// that had the worst mrr scores.
const String PRINT_WORST_RESULTS = 'print-worst-results';
/// A [Counter] to track the performance of each of the completion strategies
/// that are being compared.
@ -96,26 +127,21 @@ Counter rankComparison = Counter('relevance rank comparison');
/// Create a parser that can be used to parse the command-line arguments.
ArgParser createArgParser() {
return ArgParser()
..addFlag(AVAILABLE_SUGGESTIONS,
abbr: 'a',
help:
'Use the available suggestions feature in the Analysis Server when '
'computing the set of code completions. With this feature enabled, '
'completion will match the support in the Dart Plugin for '
'IntelliJ, without this enabled the completion support matches the '
'support in LSP.',
defaultsTo: false,
negatable: false)
..addOption(
'help',
abbr: 'h',
help: 'Print this help message.',
)
..addFlag(
VERBOSE,
abbr: 'v',
help: 'Print additional information about the analysis',
negatable: false,
)
..addFlag(AVAILABLE_SUGGESTIONS,
abbr: 'a',
help: 'Use the available suggestions feature in the Analysis Server '
'when computing the set of code completions. With this feature '
'enabled, completion will match the support in the Dart Plugin for '
'IntelliJ, without this enabled the completion support matches '
'the support in LSP.',
defaultsTo: false,
negatable: false)
..addOption(OVERLAY,
allowed: [
OVERLAY_NONE,
@ -125,9 +151,46 @@ ArgParser createArgParser() {
defaultsTo: OVERLAY_NONE,
help:
'Before attempting a completion at the location of each token, the '
'token can be removed, or the rest of the file can be removed to test '
'code completion with diverse methods. The default mode is to '
'complete at the start of the token without modifying the file.');
'token can be removed, or the rest of the file can be removed to '
'test code completion with diverse methods. The default mode is to '
'complete at the start of the token without modifying the file.')
..addFlag(PRINT_MISSED_COMPLETION_DETAILS,
defaultsTo: false,
help:
'Print detailed information every time a completion request fails '
'to produce a suggestions matching the expected suggestion.',
negatable: false)
..addFlag(PRINT_MISSED_COMPLETION_SUMMARY,
defaultsTo: false,
help: 'Print summary information about the times that a completion '
'request failed to produce a suggestions matching the expected '
'suggestion.',
negatable: false)
..addFlag(PRINT_MISSING_INFORMATION,
defaultsTo: false,
help: 'Print information about places where no completion location was '
'computed and about information that is missing in the completion '
'tables.',
negatable: false)
..addFlag(PRINT_MRR_BY_LOCATION,
defaultsTo: false,
help:
'Print information about the mrr score achieved at each completion '
'location. This can help focus efforts to improve the overall '
'score by pointing out the locations that are causing the biggest '
'impact.',
negatable: false)
..addFlag(PRINT_SLOWEST_RESULTS,
defaultsTo: false,
help: 'Print information about the completion requests that were the '
'slowest to return suggestions.',
negatable: false)
..addFlag(PRINT_WORST_RESULTS,
defaultsTo: false,
help:
'Print information about the completion requests that had the worst '
'mrr scores.',
negatable: false);
}
/// Print usage information for this tool.
@ -446,11 +509,7 @@ class CompletionMetrics {
class CompletionMetricsComputer {
final String rootPath;
final bool availableSuggestions;
final String overlay;
final bool verbose;
final CompletionMetricsOptions options;
ResolvedUnitResult _resolvedUnitResult;
@ -465,13 +524,7 @@ class CompletionMetricsComputer {
int overlayModificationStamp = 0;
CompletionMetricsComputer(this.rootPath,
{@required this.availableSuggestions,
@required this.overlay,
@required this.verbose})
: assert(overlay == OVERLAY_NONE ||
overlay == OVERLAY_REMOVE_TOKEN ||
overlay == OVERLAY_REMOVE_REST_OF_FILE);
CompletionMetricsComputer(this.rootPath, this.options);
Future<int> compute() async {
resultCode = 0;
@ -485,6 +538,7 @@ class CompletionMetricsComputer {
for (var context in collection.contexts) {
await _computeInContext(context.contextRoot);
}
printComparison();
for (var metrics in targetMetrics) {
printMetrics(metrics);
@ -493,11 +547,15 @@ class CompletionMetricsComputer {
rankComparison.printCounterValues();
print('====================');
if (verbose) {
printWorstResults(metrics);
printSlowestResults(metrics);
if (options.printMissingInformation) {
printMissingInformation(metrics);
}
if (options.printSlowestResults) {
printSlowestResults(metrics);
}
if (options.printWorstResults) {
printWorstResults(metrics);
}
}
return resultCode;
}
@ -561,12 +619,42 @@ class CompletionMetricsComputer {
}
}
void printComparison() {
List<String> toRow(Iterable<MeanReciprocalRankComputer> computers) {
return [
computers.first.name,
for (var computer in computers) (1 / computer.mrr).toStringAsFixed(3),
];
}
var buffer = StringBuffer();
var table = [
['', for (var metrics in targetMetrics) metrics.name],
toRow(targetMetrics.map((metrics) => metrics.mrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.successfulMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.instanceMemberMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.staticMemberMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.typeRefMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.localRefMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.paramRefMrrComputer)),
toRow(targetMetrics.map((metrics) => metrics.topLevelMrrComputer)),
];
for (var i = 1; i < table[0].length; i++) {
rightJustifyColumn(i, table);
}
buffer.writeTable(table);
print('');
print('Comparison of inverse mean reciprocal ranks (lower is better)');
print('');
print(buffer.toString());
}
void printMetrics(CompletionMetrics metrics) {
print('');
print('');
print('====================');
print('Completion metrics for ${metrics.name}:');
if (verbose) {
print('');
if (options.printMissedCompletionSummary) {
metrics.completionMissedTokenCounter.printCounterValues();
print('');
@ -577,31 +665,39 @@ class CompletionMetricsComputer {
print('');
}
metrics.mrrComputer.printMean();
print('');
List<String> toRow(MeanReciprocalRankComputer computer) {
return [
computer.name,
computer.mrr.toStringAsFixed(6),
(1 / computer.mrr).toStringAsFixed(3),
computer.mrr_5.toStringAsFixed(6),
(1 / computer.mrr_5).toStringAsFixed(3),
computer.count.toString(),
];
}
metrics.successfulMrrComputer.printMean();
print('');
var buffer = StringBuffer();
var table = [
['', 'mrr', 'inverse mrr', 'mrr_5', 'inverse mrr_5', 'count'],
toRow(metrics.mrrComputer),
toRow(metrics.successfulMrrComputer),
toRow(metrics.instanceMemberMrrComputer),
toRow(metrics.staticMemberMrrComputer),
toRow(metrics.typeRefMrrComputer),
toRow(metrics.localRefMrrComputer),
toRow(metrics.paramRefMrrComputer),
toRow(metrics.topLevelMrrComputer),
];
rightJustifyColumn(2, table);
rightJustifyColumn(4, table);
rightJustifyColumn(5, table);
buffer.writeTable(table);
metrics.instanceMemberMrrComputer.printMean();
print('Mean Reciprocal Rank');
print('');
print(buffer.toString());
metrics.staticMemberMrrComputer.printMean();
print('');
metrics.typeRefMrrComputer.printMean();
print('');
metrics.localRefMrrComputer.printMean();
print('');
metrics.paramRefMrrComputer.printMean();
print('');
metrics.topLevelMrrComputer.printMean();
print('');
if (verbose) {
if (options.printMrrByLocation) {
var lines = <LocationTableLine>[];
for (var entry in metrics.locationMrrComputers.entries) {
var count = entry.value.count;
@ -807,7 +903,7 @@ class CompletionMetricsComputer {
// suggestions if doComputeCompletionsFromAnalysisServer is true.
DeclarationsTracker declarationsTracker;
protocol.CompletionAvailableSuggestionsParams availableSuggestionsParams;
if (availableSuggestions) {
if (options.availableSuggestions) {
declarationsTracker = DeclarationsTracker(
MemoryByteStore(), PhysicalResourceProvider.INSTANCE);
declarationsTracker.addContext(context);
@ -852,9 +948,11 @@ class CompletionMetricsComputer {
// If an overlay option is being used, compute the overlay file, and
// have the context reanalyze the file
if (overlay != OVERLAY_NONE) {
if (options.overlay != OVERLAY_NONE) {
var overlayContents = _getOverlayContents(
_resolvedUnitResult.content, expectedCompletion, overlay);
_resolvedUnitResult.content,
expectedCompletion,
options.overlay);
_provider.setOverlay(filePath,
content: overlayContents,
@ -921,7 +1019,7 @@ class CompletionMetricsComputer {
var rank = await handleExpectedCompletion(
listener: listener,
metrics: metrics,
printMissedCompletions: verbose);
printMissedCompletions: options.printMissedCompletionDetails);
if (bestRank < 0 || rank < bestRank) {
bestRank = rank;
bestName = metrics.name;
@ -932,7 +1030,7 @@ class CompletionMetricsComputer {
// If an overlay option is being used, remove the overlay applied
// earlier
if (overlay != OVERLAY_NONE) {
if (options.overlay != OVERLAY_NONE) {
_provider.removeOverlay(filePath);
}
}
@ -1061,6 +1159,56 @@ class CompletionMetricsComputer {
}
}
/// The options specified on the command-line.
class CompletionMetricsOptions {
/// A flag indicating whether available suggestions should be enabled for this
/// run.
final bool availableSuggestions;
/// The overlay mode that should be used.
final String overlay;
/// A flag indicating whether information should be printed every time a
/// completion request fails to produce a suggestions matching the expected
/// suggestion.
final bool printMissedCompletionDetails;
/// A flag indicating whether information should be printed every time a
/// completion request fails to produce a suggestions matching the expected
/// suggestion.
final bool printMissedCompletionSummary;
/// A flag indicating whether information should be printed about places where
/// no completion location was computed and about information that's missing
/// in the completion tables.
final bool printMissingInformation;
/// A flag indicating whether information should be printed about the mrr
/// score achieved at each completion location.
final bool printMrrByLocation;
/// A flag indicating whether information should be printed about the
/// completion requests that were the slowest to return suggestions.
final bool printSlowestResults;
/// A flag indicating whether information should be printed about the
/// completion requests that had the worst mrr scores.
final bool printWorstResults;
CompletionMetricsOptions(
{@required this.availableSuggestions,
@required this.overlay,
@required this.printMissedCompletionDetails,
@required this.printMissedCompletionSummary,
@required this.printMissingInformation,
@required this.printMrrByLocation,
@required this.printSlowestResults,
@required this.printWorstResults})
: assert(overlay == OVERLAY_NONE ||
overlay == OVERLAY_REMOVE_TOKEN ||
overlay == OVERLAY_REMOVE_REST_OF_FILE);
}
/// The result of a single completion.
class CompletionResult {
final Place place;

9
pkg/analysis_server/tool/code_completion/metrics_util.dart
View file

@ -76,8 +76,13 @@ class Counter {
void printCounterValues() {
print('Counts for \'$name\' (total = $_totalCount):');
if (_totalCount > 0) {
_buckets.forEach((id, count) =>
print('[$id] $count (${printPercentage(count / _totalCount, 2)})'));
var entries = _buckets.entries.toList();
entries.sort((first, second) => second.value - first.value);
for (var entry in entries) {
var id = entry.key;
var count = entry.value;
print('[$id] $count (${printPercentage(count / _totalCount, 2)})');
}
} else {
print('<no counts>');
}

20
pkg/analysis_server/tool/code_completion/output_utilities.dart
View file

@ -4,6 +4,26 @@
import 'dart:math' as math;
/// Given a [table] represented as a list of rows, right justify all of the
/// cells in the given [column], except for the first row under the assumption
/// that the first row contains headers.
void rightJustifyColumn(int column, List<List<String>> table) {
var width = 0;
for (var i = 0; i < table.length; i++) {
var row = table[i];
width = math.max(width, row[column].length);
}
for (var i = 1; i < table.length; i++) {
var row = table[i];
var cellValue = row[column];
var length = cellValue.length;
if (length < width) {
var padding = ' ' * (width - length);
row[column] = '$padding$cellValue';
}
}
}
extension OutputUtilities on StringSink {
/// Write the given [table].
///