mirror of
https://github.com/dart-lang/sdk
synced 2024-09-16 03:27:43 +00:00
Add several Flutter completion benchmarks.
When running DAS from sources, so might be not exactly as when running from the snapshot, currently we have following numbers: Before reverting to allow available declarations: {das-flutter-completion-start: {micros: 15030209}, das-flutter-completion-smallFile-body: {micros: 619210}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 484791}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 2348386}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 3701027}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 3332297}} Current 2021-09-30 master: {das-flutter-completion-start: {micros: 15194069}, das-flutter-completion-smallFile-body: {micros: 22721}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 7796}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 552269}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 3076925}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 3010509}} Current 2021-10-01 master, with snapshot: {das-flutter-completion-start: {micros: 6657503}, das-flutter-completion-smallFile-body: {micros: 20596}, das-flutter-completion-smallFile-body-withoutPrefix: {micros: 7073}, das-flutter-completion-smallLibraryCycle-largeFile-smallBody: {micros: 556415}, das-flutter-completion-mediumLibraryCycle-mediumFile-smallBody: {micros: 85024}, das-flutter-completion-mediumLibraryCycle-mediumFile-api-parameterType: {micros: 331850}} Change-Id: I1d76f4f15d36e316b553ffc6cfaf24551c7f9bf5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/215240 Commit-Queue: Konstantin Shcheglov <scheglov@google.com> Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
This commit is contained in:
parent
4f80522fe4
commit
bc5a1138e0
|
@ -15,6 +15,7 @@ import 'package:path/path.dart' as path;
|
||||||
|
|
||||||
import 'perf/benchmarks_impl.dart';
|
import 'perf/benchmarks_impl.dart';
|
||||||
import 'perf/flutter_analyze_benchmark.dart';
|
import 'perf/flutter_analyze_benchmark.dart';
|
||||||
|
import 'perf/flutter_completion_benchmark.dart';
|
||||||
|
|
||||||
Future main(List<String> args) async {
|
Future main(List<String> args) async {
|
||||||
var benchmarks = <Benchmark>[
|
var benchmarks = <Benchmark>[
|
||||||
|
@ -23,6 +24,8 @@ Future main(List<String> args) async {
|
||||||
AnalysisBenchmark(ServerBenchmark.das),
|
AnalysisBenchmark(ServerBenchmark.das),
|
||||||
AnalysisBenchmark(ServerBenchmark.lsp),
|
AnalysisBenchmark(ServerBenchmark.lsp),
|
||||||
FlutterAnalyzeBenchmark(),
|
FlutterAnalyzeBenchmark(),
|
||||||
|
FlutterCompletionBenchmark.das,
|
||||||
|
FlutterCompletionBenchmark.lsp,
|
||||||
];
|
];
|
||||||
|
|
||||||
var runner = CommandRunner(
|
var runner = CommandRunner(
|
||||||
|
|
|
@ -0,0 +1,317 @@
|
||||||
|
// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
|
||||||
|
// for details. All rights reserved. Use of this source code is governed by a
|
||||||
|
// BSD-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
import 'dart:async';
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:path/path.dart' as path;
|
||||||
|
|
||||||
|
import '../benchmarks.dart';
|
||||||
|
import 'memory_tests.dart';
|
||||||
|
|
||||||
|
Future<int> _runProcess(
|
||||||
|
String command,
|
||||||
|
List<String> args, {
|
||||||
|
String? cwd,
|
||||||
|
bool failOnError = true,
|
||||||
|
}) async {
|
||||||
|
print('\n$command ${args.join(' ')}');
|
||||||
|
|
||||||
|
var process = await Process.start(command, args, workingDirectory: cwd);
|
||||||
|
|
||||||
|
process.stdout
|
||||||
|
.transform(utf8.decoder)
|
||||||
|
.transform(LineSplitter())
|
||||||
|
.listen((line) {
|
||||||
|
print(' $line');
|
||||||
|
});
|
||||||
|
process.stderr
|
||||||
|
.transform(utf8.decoder)
|
||||||
|
.transform(LineSplitter())
|
||||||
|
.listen((line) => print(' $line'));
|
||||||
|
|
||||||
|
var exitCode = await process.exitCode;
|
||||||
|
if (exitCode != 0 && failOnError) {
|
||||||
|
throw '$command exited with $exitCode';
|
||||||
|
}
|
||||||
|
|
||||||
|
return exitCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// benchmarks:
|
||||||
|
/// - analysis-server-warm-analysis
|
||||||
|
/// - analysis-server-warm-memory
|
||||||
|
/// - analysis-server-edit
|
||||||
|
/// - analysis-server-completion
|
||||||
|
class FlutterCompletionBenchmark extends Benchmark {
|
||||||
|
static final das = FlutterCompletionBenchmark(
|
||||||
|
'das',
|
||||||
|
() => AnalysisServerBenchmarkTest(),
|
||||||
|
);
|
||||||
|
|
||||||
|
static final lsp = FlutterCompletionBenchmark(
|
||||||
|
'lsp',
|
||||||
|
() => LspAnalysisServerBenchmarkTest(),
|
||||||
|
);
|
||||||
|
|
||||||
|
final AbstractBenchmarkTest Function() testConstructor;
|
||||||
|
|
||||||
|
late String flutterPath;
|
||||||
|
|
||||||
|
FlutterCompletionBenchmark(String protocolName, this.testConstructor)
|
||||||
|
: super(
|
||||||
|
'$protocolName-flutter-completion',
|
||||||
|
'Completion benchmarks with Flutter.',
|
||||||
|
kind: 'group',
|
||||||
|
);
|
||||||
|
|
||||||
|
@override
|
||||||
|
bool get needsSetup => true;
|
||||||
|
|
||||||
|
@override
|
||||||
|
Future oneTimeSetup() async {
|
||||||
|
flutterPath = Directory.systemTemp.createTempSync('flutter').path;
|
||||||
|
|
||||||
|
// git clone https://github.com/flutter/flutter $flutterDir
|
||||||
|
await _runProcess('git', [
|
||||||
|
'clone',
|
||||||
|
'https://github.com/flutter/flutter',
|
||||||
|
path.canonicalize(flutterPath)
|
||||||
|
]);
|
||||||
|
|
||||||
|
var flutterTool = path.join(flutterPath, 'bin', 'flutter');
|
||||||
|
|
||||||
|
// flutter --version
|
||||||
|
await _runProcess(flutterTool, ['--version'], cwd: flutterPath);
|
||||||
|
|
||||||
|
// flutter update-packages
|
||||||
|
await _runProcess(
|
||||||
|
flutterTool,
|
||||||
|
['pub', 'get'],
|
||||||
|
cwd: path.join(flutterPath, 'packages', 'flutter'),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
Future<BenchMarkResult> run({
|
||||||
|
bool quick = false,
|
||||||
|
bool verbose = false,
|
||||||
|
}) async {
|
||||||
|
if (!quick) {
|
||||||
|
deleteServerCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
var test = testConstructor();
|
||||||
|
if (verbose) {
|
||||||
|
test.debugStdio();
|
||||||
|
}
|
||||||
|
|
||||||
|
final flutterPkgPath = path.join(flutterPath, 'packages', 'flutter');
|
||||||
|
|
||||||
|
// Open a small directory, but with the package config that allows us
|
||||||
|
// to analyze any file in `package:flutter`, including tests.
|
||||||
|
var startTimer = Stopwatch()..start();
|
||||||
|
await test.setUp([
|
||||||
|
'$flutterPkgPath/lib/src/physics',
|
||||||
|
]);
|
||||||
|
|
||||||
|
await test.analysisFinished;
|
||||||
|
startTimer.stop();
|
||||||
|
|
||||||
|
var result = CompoundBenchMarkResult(id);
|
||||||
|
result.add(
|
||||||
|
'start',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
startTimer.elapsedMicroseconds,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
// This is a scenario of an easy case - the file is small, less than
|
||||||
|
// 3KB, and we insert a prefix with a good selectivity. So, everything
|
||||||
|
// should be fast. We should just make sure to don't spend too much
|
||||||
|
// time analyzing, and do apply the filter.
|
||||||
|
// Total number of suggestions: 2322.
|
||||||
|
// Filtered to: 82.
|
||||||
|
result.add(
|
||||||
|
'smallFile-body',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
await _completionTiming(
|
||||||
|
test,
|
||||||
|
filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart',
|
||||||
|
uniquePrefix: 'Widget build(BuildContext context) {',
|
||||||
|
insertStringGenerator: () => 'M',
|
||||||
|
name: 'smallFile-body',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!quick) {
|
||||||
|
// This scenario is relatively easy - the file is small, less then 3KB.
|
||||||
|
// But we don't have any prefix to filter, so if we don't restrict the
|
||||||
|
// number of suggestions, we might spend too much time serializing into
|
||||||
|
// JSON in the server, and deserializing on the client.
|
||||||
|
// Total number of suggestions: 2322.
|
||||||
|
// Filtered to: 2322.
|
||||||
|
result.add(
|
||||||
|
'smallFile-body-withoutPrefix',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
await _completionTiming(
|
||||||
|
test,
|
||||||
|
filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart',
|
||||||
|
uniquePrefix: 'Widget build(BuildContext context) {',
|
||||||
|
insertStringGenerator: null,
|
||||||
|
name: 'smallFile-body-withoutPrefix',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
// The key part of this scenario is that we work in a relatively large
|
||||||
|
// file, about 340KB. So, it is expensive to parse and resolve. And
|
||||||
|
// we simulate changing it by typing a prefix, as users often do.
|
||||||
|
// The target method body is small, so something could be optimized.
|
||||||
|
// Total number of suggestions: 4654.
|
||||||
|
// Filtered to: 182.
|
||||||
|
result.add(
|
||||||
|
'smallLibraryCycle-largeFile-smallBody',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
await _completionTiming(
|
||||||
|
test,
|
||||||
|
filePath: '$flutterPkgPath/test/material/text_field_test.dart',
|
||||||
|
uniquePrefix: 'getOpacity(WidgetTester tester, Finder finder) {',
|
||||||
|
insertStringGenerator: () => 'M',
|
||||||
|
name: 'smallLibraryCycle-largeFile-smallBody',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
// In this scenario we change a file that is in a library cycle
|
||||||
|
// with 69 libraries. So, the implementation might discard information
|
||||||
|
// about all these libraries. We change a method body, so the API
|
||||||
|
// signature is the same, and we are able to reload these libraries
|
||||||
|
// from bytes. But this still costs something.
|
||||||
|
// There is also a spill-over from the previous test - we send a lot
|
||||||
|
// (about 5MB) of available declarations after each change. This makes
|
||||||
|
// completion response times very large.
|
||||||
|
// TODO(scheglov) Remove the previous sentence when improved.
|
||||||
|
// Total number of suggestions: 3429.
|
||||||
|
// Filtered to: 133.
|
||||||
|
result.add(
|
||||||
|
'mediumLibraryCycle-mediumFile-smallBody',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
await _completionTiming(
|
||||||
|
test,
|
||||||
|
filePath: '$flutterPkgPath/lib/src/material/app_bar.dart',
|
||||||
|
uniquePrefix: 'computeDryLayout(BoxConstraints constraints) {',
|
||||||
|
insertStringGenerator: () => 'M',
|
||||||
|
name: 'mediumLibraryCycle-mediumFile-smallBody',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
// In this scenario is that we change a file that is in a library cycle
|
||||||
|
// with 69 libraries. Moreover, we change the API - the type of a
|
||||||
|
// formal parameter. So, potentially we need to relink the whole library
|
||||||
|
// cycle. This is expensive.
|
||||||
|
// Total number of suggestions: 1510.
|
||||||
|
// Filtered to: 0.
|
||||||
|
result.add(
|
||||||
|
'mediumLibraryCycle-mediumFile-api-parameterType',
|
||||||
|
BenchMarkResult(
|
||||||
|
'micros',
|
||||||
|
await _completionTiming(
|
||||||
|
test,
|
||||||
|
filePath: '$flutterPkgPath/lib/src/material/app_bar.dart',
|
||||||
|
uniquePrefix: 'computeDryLayout(BoxConstraints',
|
||||||
|
insertStringGenerator: _IncrementingStringGenerator(),
|
||||||
|
name: 'mediumLibraryCycle-mediumFile-api-parameterType',
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await test.shutdown();
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Perform completion in [filePath] at the end of the [uniquePrefix].
|
||||||
|
///
|
||||||
|
/// If [insertStringGenerator] is not `null`, insert it, and complete after
|
||||||
|
/// it. So, we can simulate user typing to start completion.
|
||||||
|
Future<int> _completionTiming(
|
||||||
|
AbstractBenchmarkTest test, {
|
||||||
|
required String filePath,
|
||||||
|
required String uniquePrefix,
|
||||||
|
required String Function()? insertStringGenerator,
|
||||||
|
String? name,
|
||||||
|
}) async {
|
||||||
|
final fileContent = File(filePath).readAsStringSync();
|
||||||
|
|
||||||
|
final prefixOffset = fileContent.indexOf(uniquePrefix);
|
||||||
|
if (prefixOffset == -1) {
|
||||||
|
throw StateError('Cannot find: $uniquePrefix');
|
||||||
|
}
|
||||||
|
if (fileContent.contains(uniquePrefix, prefixOffset + 1)) {
|
||||||
|
throw StateError('Not unique: $uniquePrefix');
|
||||||
|
}
|
||||||
|
|
||||||
|
final prefixEnd = prefixOffset + uniquePrefix.length;
|
||||||
|
|
||||||
|
await test.openFile(filePath, fileContent);
|
||||||
|
|
||||||
|
Future<void> perform() async {
|
||||||
|
var completionOffset = prefixEnd;
|
||||||
|
|
||||||
|
if (insertStringGenerator != null) {
|
||||||
|
final insertString = insertStringGenerator();
|
||||||
|
completionOffset += insertString.length;
|
||||||
|
var newCode = fileContent.substring(0, prefixEnd) +
|
||||||
|
insertString +
|
||||||
|
fileContent.substring(prefixEnd);
|
||||||
|
await test.updateFile(
|
||||||
|
filePath,
|
||||||
|
newCode,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await test.complete(filePath, completionOffset);
|
||||||
|
|
||||||
|
if (insertStringGenerator != null) {
|
||||||
|
await test.updateFile(filePath, fileContent);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Perform warm-up.
|
||||||
|
// The cold start does not matter.
|
||||||
|
// The sustained performance is much more important.
|
||||||
|
const kWarmUpCount = 50;
|
||||||
|
for (var i = 0; i < kWarmUpCount; i++) {
|
||||||
|
await perform();
|
||||||
|
}
|
||||||
|
|
||||||
|
const kRepeatCount = 10;
|
||||||
|
final timer = Stopwatch()..start();
|
||||||
|
for (var i = 0; i < kRepeatCount; i++) {
|
||||||
|
await perform();
|
||||||
|
}
|
||||||
|
|
||||||
|
await test.closeFile(filePath);
|
||||||
|
|
||||||
|
return timer.elapsedMicroseconds ~/ kRepeatCount;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class _IncrementingStringGenerator {
|
||||||
|
int _value = 0;
|
||||||
|
|
||||||
|
String call() {
|
||||||
|
return '${_value++}';
|
||||||
|
}
|
||||||
|
}
|
|
@ -79,6 +79,7 @@ class AnalysisServerBenchmarkTest extends AbstractBenchmarkTest {
|
||||||
Future<void> setUp(List<String> roots) async {
|
Future<void> setUp(List<String> roots) async {
|
||||||
await _test.setUp();
|
await _test.setUp();
|
||||||
await _test.subscribeToStatusNotifications();
|
await _test.subscribeToStatusNotifications();
|
||||||
|
await _test.subscribeToAvailableSuggestions();
|
||||||
await _test.sendAnalysisSetAnalysisRoots(roots, []);
|
await _test.sendAnalysisSetAnalysisRoots(roots, []);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -127,6 +128,16 @@ class AnalysisServerMemoryUsageTest
|
||||||
/// After every test, the server is stopped.
|
/// After every test, the server is stopped.
|
||||||
Future shutdown() async => await shutdownIfNeeded();
|
Future shutdown() async => await shutdownIfNeeded();
|
||||||
|
|
||||||
|
/// Enable using available suggestions during completion.
|
||||||
|
Future<void> subscribeToAvailableSuggestions() async {
|
||||||
|
await server.send(
|
||||||
|
'completion.setSubscriptions',
|
||||||
|
CompletionSetSubscriptionsParams(
|
||||||
|
[CompletionService.AVAILABLE_SUGGESTION_SETS],
|
||||||
|
).toJson(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// Enable [ServerService.STATUS] notifications so that [analysisFinished]
|
/// Enable [ServerService.STATUS] notifications so that [analysisFinished]
|
||||||
/// can be used.
|
/// can be used.
|
||||||
Future subscribeToStatusNotifications() async {
|
Future subscribeToStatusNotifications() async {
|
||||||
|
|
Loading…
Reference in a new issue