mirror of
https://github.com/dart-lang/sdk
synced 2024-09-05 16:41:07 +00:00
Add several Flutter completion benchmarks.
When running DAS from sources, so might be not exactly as when running from the snapshot, currently we have following numbers: Before reverting to allow available declarations: {das-flutter-completion-start: {micros: 15030209}, das-flutter-completion-smallFile-body: {micros: 619210}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 484791}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 2348386}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 3701027}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 3332297}} Current 2021-09-30 master: {das-flutter-completion-start: {micros: 15194069}, das-flutter-completion-smallFile-body: {micros: 22721}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 7796}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 552269}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 3076925}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 3010509}} Current 2021-10-01 master, with snapshot: {das-flutter-completion-start: {micros: 6657503}, das-flutter-completion-smallFile-body: {micros: 20596}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 7073}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 556415}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 85024}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 331850}} Change-Id: I1d76f4f15d36e316b553ffc6cfaf24551c7f9bf5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/215240 Commit-Queue: Konstantin Shcheglov <scheglov@google.com> Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
parent
4f80522fe4
commit
bc5a1138e0
|
@ -15,6 +15,7 @@ import 'package:path/path.dart' as path;
|
|||
|
||||
import 'perf/benchmarks_impl.dart';
|
||||
import 'perf/flutter_analyze_benchmark.dart';
|
||||
import 'perf/flutter_completion_benchmark.dart';
|
||||
|
||||
Future main(List<String> args) async {
|
||||
var benchmarks = <Benchmark>[
|
||||
|
@ -23,6 +24,8 @@ Future main(List<String> args) async {
|
|||
AnalysisBenchmark(ServerBenchmark.das),
|
||||
AnalysisBenchmark(ServerBenchmark.lsp),
|
||||
FlutterAnalyzeBenchmark(),
|
||||
FlutterCompletionBenchmark.das,
|
||||
FlutterCompletionBenchmark.lsp,
|
||||
];
|
||||
|
||||
var runner = CommandRunner(
|
||||
|
|
|
@ -0,0 +1,317 @@
|
|||
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
|
||||
// for details. All rights reserved. Use of this source code is governed by a
|
||||
// BSD-style license that can be found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../benchmarks.dart';
|
||||
import 'memory_tests.dart';
|
||||
|
||||
Future<int> _runProcess(
|
||||
String command,
|
||||
List<String> args, {
|
||||
String? cwd,
|
||||
bool failOnError = true,
|
||||
}) async {
|
||||
print('\n$command ${args.join(' ')}');
|
||||
|
||||
var process = await Process.start(command, args, workingDirectory: cwd);
|
||||
|
||||
process.stdout
|
||||
.transform(utf8.decoder)
|
||||
.transform(LineSplitter())
|
||||
.listen((line) {
|
||||
print(' $line');
|
||||
});
|
||||
process.stderr
|
||||
.transform(utf8.decoder)
|
||||
.transform(LineSplitter())
|
||||
.listen((line) => print(' $line'));
|
||||
|
||||
var exitCode = await process.exitCode;
|
||||
if (exitCode != 0 && failOnError) {
|
||||
throw '$command exited with $exitCode';
|
||||
}
|
||||
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
/// benchmarks:
|
||||
/// - analysis-server-warm-analysis
|
||||
/// - analysis-server-warm-memory
|
||||
/// - analysis-server-edit
|
||||
/// - analysis-server-completion
|
||||
class FlutterCompletionBenchmark extends Benchmark {
|
||||
static final das = FlutterCompletionBenchmark(
|
||||
'das',
|
||||
() => AnalysisServerBenchmarkTest(),
|
||||
);
|
||||
|
||||
static final lsp = FlutterCompletionBenchmark(
|
||||
'lsp',
|
||||
() => LspAnalysisServerBenchmarkTest(),
|
||||
);
|
||||
|
||||
final AbstractBenchmarkTest Function() testConstructor;
|
||||
|
||||
late String flutterPath;
|
||||
|
||||
FlutterCompletionBenchmark(String protocolName, this.testConstructor)
|
||||
: super(
|
||||
'$protocolName-flutter-completion',
|
||||
'Completion benchmarks with Flutter.',
|
||||
kind: 'group',
|
||||
);
|
||||
|
||||
@override
|
||||
bool get needsSetup => true;
|
||||
|
||||
@override
|
||||
Future oneTimeSetup() async {
|
||||
flutterPath = Directory.systemTemp.createTempSync('flutter').path;
|
||||
|
||||
// git clone https://github.com/flutter/flutter $flutterDir
|
||||
await _runProcess('git', [
|
||||
'clone',
|
||||
'https://github.com/flutter/flutter',
|
||||
path.canonicalize(flutterPath)
|
||||
]);
|
||||
|
||||
var flutterTool = path.join(flutterPath, 'bin', 'flutter');
|
||||
|
||||
// flutter --version
|
||||
await _runProcess(flutterTool, ['--version'], cwd: flutterPath);
|
||||
|
||||
// flutter update-packages
|
||||
await _runProcess(
|
||||
flutterTool,
|
||||
['pub', 'get'],
|
||||
cwd: path.join(flutterPath, 'packages', 'flutter'),
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
Future<BenchMarkResult> run({
|
||||
bool quick = false,
|
||||
bool verbose = false,
|
||||
}) async {
|
||||
if (!quick) {
|
||||
deleteServerCache();
|
||||
}
|
||||
|
||||
var test = testConstructor();
|
||||
if (verbose) {
|
||||
test.debugStdio();
|
||||
}
|
||||
|
||||
final flutterPkgPath = path.join(flutterPath, 'packages', 'flutter');
|
||||
|
||||
// Open a small directory, but with the package config that allows us
|
||||
// to analyze any file in `package:flutter`, including tests.
|
||||
var startTimer = Stopwatch()..start();
|
||||
await test.setUp([
|
||||
'$flutterPkgPath/lib/src/physics',
|
||||
]);
|
||||
|
||||
await test.analysisFinished;
|
||||
startTimer.stop();
|
||||
|
||||
var result = CompoundBenchMarkResult(id);
|
||||
result.add(
|
||||
'start',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
startTimer.elapsedMicroseconds,
|
||||
),
|
||||
);
|
||||
|
||||
// This is a scenario of an easy case - the file is small, less than
|
||||
// 3KB, and we insert a prefix with a good selectivity. So, everything
|
||||
// should be fast. We should just make sure to don't spend too much
|
||||
// time analyzing, and do apply the filter.
|
||||
// Total number of suggestions: 2322.
|
||||
// Filtered to: 82.
|
||||
result.add(
|
||||
'smallFile-body',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
await _completionTiming(
|
||||
test,
|
||||
filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart',
|
||||
uniquePrefix: 'Widget build(BuildContext context) {',
|
||||
insertStringGenerator: () => 'M',
|
||||
name: 'smallFile-body',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
if (!quick) {
|
||||
// This scenario is relatively easy - the file is small, less then 3KB.
|
||||
// But we don't have any prefix to filter, so if we don't restrict the
|
||||
// number of suggestions, we might spend too much time serializing into
|
||||
// JSON in the server, and deserializing on the client.
|
||||
// Total number of suggestions: 2322.
|
||||
// Filtered to: 2322.
|
||||
result.add(
|
||||
'smallFile-body-withoutPrefix',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
await _completionTiming(
|
||||
test,
|
||||
filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart',
|
||||
uniquePrefix: 'Widget build(BuildContext context) {',
|
||||
insertStringGenerator: null,
|
||||
name: 'smallFile-body-withoutPrefix',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// The key part of this scenario is that we work in a relatively large
|
||||
// file, about 340KB. So, it is expensive to parse and resolve. And
|
||||
// we simulate changing it by typing a prefix, as users often do.
|
||||
// The target method body is small, so something could be optimized.
|
||||
// Total number of suggestions: 4654.
|
||||
// Filtered to: 182.
|
||||
result.add(
|
||||
'smallLibraryCycle-largeFile-smallBody',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
await _completionTiming(
|
||||
test,
|
||||
filePath: '$flutterPkgPath/test/material/text_field_test.dart',
|
||||
uniquePrefix: 'getOpacity(WidgetTester tester, Finder finder) {',
|
||||
insertStringGenerator: () => 'M',
|
||||
name: 'smallLibraryCycle-largeFile-smallBody',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// In this scenario we change a file that is in a library cycle
|
||||
// with 69 libraries. So, the implementation might discard information
|
||||
// about all these libraries. We change a method body, so the API
|
||||
// signature is the same, and we are able to reload these libraries
|
||||
// from bytes. But this still costs something.
|
||||
// There is also a spill-over from the previous test - we send a lot
|
||||
// (about 5MB) of available declarations after each change. This makes
|
||||
// completion response times very large.
|
||||
// TODO(scheglov) Remove the previous sentence when improved.
|
||||
// Total number of suggestions: 3429.
|
||||
// Filtered to: 133.
|
||||
result.add(
|
||||
'mediumLibraryCycle-mediumFile-smallBody',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
await _completionTiming(
|
||||
test,
|
||||
filePath: '$flutterPkgPath/lib/src/material/app_bar.dart',
|
||||
uniquePrefix: 'computeDryLayout(BoxConstraints constraints) {',
|
||||
insertStringGenerator: () => 'M',
|
||||
name: 'mediumLibraryCycle-mediumFile-smallBody',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// In this scenario is that we change a file that is in a library cycle
|
||||
// with 69 libraries. Moreover, we change the API - the type of a
|
||||
// formal parameter. So, potentially we need to relink the whole library
|
||||
// cycle. This is expensive.
|
||||
// Total number of suggestions: 1510.
|
||||
// Filtered to: 0.
|
||||
result.add(
|
||||
'mediumLibraryCycle-mediumFile-api-parameterType',
|
||||
BenchMarkResult(
|
||||
'micros',
|
||||
await _completionTiming(
|
||||
test,
|
||||
filePath: '$flutterPkgPath/lib/src/material/app_bar.dart',
|
||||
uniquePrefix: 'computeDryLayout(BoxConstraints',
|
||||
insertStringGenerator: _IncrementingStringGenerator(),
|
||||
name: 'mediumLibraryCycle-mediumFile-api-parameterType',
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
await test.shutdown();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Perform completion in [filePath] at the end of the [uniquePrefix].
|
||||
///
|
||||
/// If [insertStringGenerator] is not `null`, insert it, and complete after
|
||||
/// it. So, we can simulate user typing to start completion.
|
||||
Future<int> _completionTiming(
|
||||
AbstractBenchmarkTest test, {
|
||||
required String filePath,
|
||||
required String uniquePrefix,
|
||||
required String Function()? insertStringGenerator,
|
||||
String? name,
|
||||
}) async {
|
||||
final fileContent = File(filePath).readAsStringSync();
|
||||
|
||||
final prefixOffset = fileContent.indexOf(uniquePrefix);
|
||||
if (prefixOffset == -1) {
|
||||
throw StateError('Cannot find: $uniquePrefix');
|
||||
}
|
||||
if (fileContent.contains(uniquePrefix, prefixOffset + 1)) {
|
||||
throw StateError('Not unique: $uniquePrefix');
|
||||
}
|
||||
|
||||
final prefixEnd = prefixOffset + uniquePrefix.length;
|
||||
|
||||
await test.openFile(filePath, fileContent);
|
||||
|
||||
Future<void> perform() async {
|
||||
var completionOffset = prefixEnd;
|
||||
|
||||
if (insertStringGenerator != null) {
|
||||
final insertString = insertStringGenerator();
|
||||
completionOffset += insertString.length;
|
||||
var newCode = fileContent.substring(0, prefixEnd) +
|
||||
insertString +
|
||||
fileContent.substring(prefixEnd);
|
||||
await test.updateFile(
|
||||
filePath,
|
||||
newCode,
|
||||
);
|
||||
}
|
||||
|
||||
await test.complete(filePath, completionOffset);
|
||||
|
||||
if (insertStringGenerator != null) {
|
||||
await test.updateFile(filePath, fileContent);
|
||||
}
|
||||
}
|
||||
|
||||
// Perform warm-up.
|
||||
// The cold start does not matter.
|
||||
// The sustained performance is much more important.
|
||||
const kWarmUpCount = 50;
|
||||
for (var i = 0; i < kWarmUpCount; i++) {
|
||||
await perform();
|
||||
}
|
||||
|
||||
const kRepeatCount = 10;
|
||||
final timer = Stopwatch()..start();
|
||||
for (var i = 0; i < kRepeatCount; i++) {
|
||||
await perform();
|
||||
}
|
||||
|
||||
await test.closeFile(filePath);
|
||||
|
||||
return timer.elapsedMicroseconds ~/ kRepeatCount;
|
||||
}
|
||||
}
|
||||
|
||||
class _IncrementingStringGenerator {
|
||||
int _value = 0;
|
||||
|
||||
String call() {
|
||||
return '${_value++}';
|
||||
}
|
||||
}
|
|
@ -79,6 +79,7 @@ class AnalysisServerBenchmarkTest extends AbstractBenchmarkTest {
|
|||
Future<void> setUp(List<String> roots) async {
|
||||
await _test.setUp();
|
||||
await _test.subscribeToStatusNotifications();
|
||||
await _test.subscribeToAvailableSuggestions();
|
||||
await _test.sendAnalysisSetAnalysisRoots(roots, []);
|
||||
}
|
||||
|
||||
|
@ -127,6 +128,16 @@ class AnalysisServerMemoryUsageTest
|
|||
/// After every test, the server is stopped.
|
||||
Future shutdown() async => await shutdownIfNeeded();
|
||||
|
||||
/// Enable using available suggestions during completion.
|
||||
Future<void> subscribeToAvailableSuggestions() async {
|
||||
await server.send(
|
||||
'completion.setSubscriptions',
|
||||
CompletionSetSubscriptionsParams(
|
||||
[CompletionService.AVAILABLE_SUGGESTION_SETS],
|
||||
).toJson(),
|
||||
);
|
||||
}
|
||||
|
||||
/// Enable [ServerService.STATUS] notifications so that [analysisFinished]
|
||||
/// can be used.
|
||||
Future subscribeToStatusNotifications() async {
|
||||
|
|
Loading…
Reference in a new issue