diff --git a/pkg/analysis_server/benchmark/benchmarks.dart b/pkg/analysis_server/benchmark/benchmarks.dart index 28c658d6fcf..a92b60e972a 100644 --- a/pkg/analysis_server/benchmark/benchmarks.dart +++ b/pkg/analysis_server/benchmark/benchmarks.dart @@ -15,6 +15,7 @@ import 'package:path/path.dart' as path; import 'perf/benchmarks_impl.dart'; import 'perf/flutter_analyze_benchmark.dart'; +import 'perf/flutter_completion_benchmark.dart'; Future main(List args) async { var benchmarks = [ @@ -23,6 +24,8 @@ Future main(List args) async { AnalysisBenchmark(ServerBenchmark.das), AnalysisBenchmark(ServerBenchmark.lsp), FlutterAnalyzeBenchmark(), + FlutterCompletionBenchmark.das, + FlutterCompletionBenchmark.lsp, ]; var runner = CommandRunner( diff --git a/pkg/analysis_server/benchmark/perf/flutter_completion_benchmark.dart b/pkg/analysis_server/benchmark/perf/flutter_completion_benchmark.dart new file mode 100644 index 00000000000..1784e96c33a --- /dev/null +++ b/pkg/analysis_server/benchmark/perf/flutter_completion_benchmark.dart @@ -0,0 +1,317 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import '../benchmarks.dart'; +import 'memory_tests.dart'; + +Future _runProcess( + String command, + List args, { + String? cwd, + bool failOnError = true, +}) async { + print('\n$command ${args.join(' ')}'); + + var process = await Process.start(command, args, workingDirectory: cwd); + + process.stdout + .transform(utf8.decoder) + .transform(LineSplitter()) + .listen((line) { + print(' $line'); + }); + process.stderr + .transform(utf8.decoder) + .transform(LineSplitter()) + .listen((line) => print(' $line')); + + var exitCode = await process.exitCode; + if (exitCode != 0 && failOnError) { + throw '$command exited with $exitCode'; + } + + return exitCode; +} + +/// benchmarks: +/// - analysis-server-warm-analysis +/// - analysis-server-warm-memory +/// - analysis-server-edit +/// - analysis-server-completion +class FlutterCompletionBenchmark extends Benchmark { + static final das = FlutterCompletionBenchmark( + 'das', + () => AnalysisServerBenchmarkTest(), + ); + + static final lsp = FlutterCompletionBenchmark( + 'lsp', + () => LspAnalysisServerBenchmarkTest(), + ); + + final AbstractBenchmarkTest Function() testConstructor; + + late String flutterPath; + + FlutterCompletionBenchmark(String protocolName, this.testConstructor) + : super( + '$protocolName-flutter-completion', + 'Completion benchmarks with Flutter.', + kind: 'group', + ); + + @override + bool get needsSetup => true; + + @override + Future oneTimeSetup() async { + flutterPath = Directory.systemTemp.createTempSync('flutter').path; + + // git clone https://github.com/flutter/flutter $flutterDir + await _runProcess('git', [ + 'clone', + 'https://github.com/flutter/flutter', + path.canonicalize(flutterPath) + ]); + + var flutterTool = path.join(flutterPath, 'bin', 'flutter'); + + // flutter --version + await _runProcess(flutterTool, ['--version'], cwd: flutterPath); + + // flutter update-packages + await _runProcess( + flutterTool, + ['pub', 'get'], + cwd: path.join(flutterPath, 'packages', 'flutter'), + ); + } + + @override + Future run({ + bool quick = false, + bool verbose = false, + }) async { + if (!quick) { + deleteServerCache(); + } + + var test = testConstructor(); + if (verbose) { + test.debugStdio(); + } + + final flutterPkgPath = path.join(flutterPath, 'packages', 'flutter'); + + // Open a small directory, but with the package config that allows us + // to analyze any file in `package:flutter`, including tests. + var startTimer = Stopwatch()..start(); + await test.setUp([ + '$flutterPkgPath/lib/src/physics', + ]); + + await test.analysisFinished; + startTimer.stop(); + + var result = CompoundBenchMarkResult(id); + result.add( + 'start', + BenchMarkResult( + 'micros', + startTimer.elapsedMicroseconds, + ), + ); + + // This is a scenario of an easy case - the file is small, less than + // 3KB, and we insert a prefix with a good selectivity. So, everything + // should be fast. We should just make sure to don't spend too much + // time analyzing, and do apply the filter. + // Total number of suggestions: 2322. + // Filtered to: 82. + result.add( + 'smallFile-body', + BenchMarkResult( + 'micros', + await _completionTiming( + test, + filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart', + uniquePrefix: 'Widget build(BuildContext context) {', + insertStringGenerator: () => 'M', + name: 'smallFile-body', + ), + ), + ); + + if (!quick) { + // This scenario is relatively easy - the file is small, less then 3KB. + // But we don't have any prefix to filter, so if we don't restrict the + // number of suggestions, we might spend too much time serializing into + // JSON in the server, and deserializing on the client. + // Total number of suggestions: 2322. + // Filtered to: 2322. + result.add( + 'smallFile-body-withoutPrefix', + BenchMarkResult( + 'micros', + await _completionTiming( + test, + filePath: '$flutterPkgPath/lib/src/material/flutter_logo.dart', + uniquePrefix: 'Widget build(BuildContext context) {', + insertStringGenerator: null, + name: 'smallFile-body-withoutPrefix', + ), + ), + ); + + // The key part of this scenario is that we work in a relatively large + // file, about 340KB. So, it is expensive to parse and resolve. And + // we simulate changing it by typing a prefix, as users often do. + // The target method body is small, so something could be optimized. + // Total number of suggestions: 4654. + // Filtered to: 182. + result.add( + 'smallLibraryCycle-largeFile-smallBody', + BenchMarkResult( + 'micros', + await _completionTiming( + test, + filePath: '$flutterPkgPath/test/material/text_field_test.dart', + uniquePrefix: 'getOpacity(WidgetTester tester, Finder finder) {', + insertStringGenerator: () => 'M', + name: 'smallLibraryCycle-largeFile-smallBody', + ), + ), + ); + + // In this scenario we change a file that is in a library cycle + // with 69 libraries. So, the implementation might discard information + // about all these libraries. We change a method body, so the API + // signature is the same, and we are able to reload these libraries + // from bytes. But this still costs something. + // There is also a spill-over from the previous test - we send a lot + // (about 5MB) of available declarations after each change. This makes + // completion response times very large. + // TODO(scheglov) Remove the previous sentence when improved. + // Total number of suggestions: 3429. + // Filtered to: 133. + result.add( + 'mediumLibraryCycle-mediumFile-smallBody', + BenchMarkResult( + 'micros', + await _completionTiming( + test, + filePath: '$flutterPkgPath/lib/src/material/app_bar.dart', + uniquePrefix: 'computeDryLayout(BoxConstraints constraints) {', + insertStringGenerator: () => 'M', + name: 'mediumLibraryCycle-mediumFile-smallBody', + ), + ), + ); + + // In this scenario is that we change a file that is in a library cycle + // with 69 libraries. Moreover, we change the API - the type of a + // formal parameter. So, potentially we need to relink the whole library + // cycle. This is expensive. + // Total number of suggestions: 1510. + // Filtered to: 0. + result.add( + 'mediumLibraryCycle-mediumFile-api-parameterType', + BenchMarkResult( + 'micros', + await _completionTiming( + test, + filePath: '$flutterPkgPath/lib/src/material/app_bar.dart', + uniquePrefix: 'computeDryLayout(BoxConstraints', + insertStringGenerator: _IncrementingStringGenerator(), + name: 'mediumLibraryCycle-mediumFile-api-parameterType', + ), + ), + ); + } + + await test.shutdown(); + + return result; + } + + /// Perform completion in [filePath] at the end of the [uniquePrefix]. + /// + /// If [insertStringGenerator] is not `null`, insert it, and complete after + /// it. So, we can simulate user typing to start completion. + Future _completionTiming( + AbstractBenchmarkTest test, { + required String filePath, + required String uniquePrefix, + required String Function()? insertStringGenerator, + String? name, + }) async { + final fileContent = File(filePath).readAsStringSync(); + + final prefixOffset = fileContent.indexOf(uniquePrefix); + if (prefixOffset == -1) { + throw StateError('Cannot find: $uniquePrefix'); + } + if (fileContent.contains(uniquePrefix, prefixOffset + 1)) { + throw StateError('Not unique: $uniquePrefix'); + } + + final prefixEnd = prefixOffset + uniquePrefix.length; + + await test.openFile(filePath, fileContent); + + Future perform() async { + var completionOffset = prefixEnd; + + if (insertStringGenerator != null) { + final insertString = insertStringGenerator(); + completionOffset += insertString.length; + var newCode = fileContent.substring(0, prefixEnd) + + insertString + + fileContent.substring(prefixEnd); + await test.updateFile( + filePath, + newCode, + ); + } + + await test.complete(filePath, completionOffset); + + if (insertStringGenerator != null) { + await test.updateFile(filePath, fileContent); + } + } + + // Perform warm-up. + // The cold start does not matter. + // The sustained performance is much more important. + const kWarmUpCount = 50; + for (var i = 0; i < kWarmUpCount; i++) { + await perform(); + } + + const kRepeatCount = 10; + final timer = Stopwatch()..start(); + for (var i = 0; i < kRepeatCount; i++) { + await perform(); + } + + await test.closeFile(filePath); + + return timer.elapsedMicroseconds ~/ kRepeatCount; + } +} + +class _IncrementingStringGenerator { + int _value = 0; + + String call() { + return '${_value++}'; + } +} diff --git a/pkg/analysis_server/benchmark/perf/memory_tests.dart b/pkg/analysis_server/benchmark/perf/memory_tests.dart index b67b63c1246..528f6725bb9 100644 --- a/pkg/analysis_server/benchmark/perf/memory_tests.dart +++ b/pkg/analysis_server/benchmark/perf/memory_tests.dart @@ -79,6 +79,7 @@ class AnalysisServerBenchmarkTest extends AbstractBenchmarkTest { Future setUp(List roots) async { await _test.setUp(); await _test.subscribeToStatusNotifications(); + await _test.subscribeToAvailableSuggestions(); await _test.sendAnalysisSetAnalysisRoots(roots, []); } @@ -127,6 +128,16 @@ class AnalysisServerMemoryUsageTest /// After every test, the server is stopped. Future shutdown() async => await shutdownIfNeeded(); + /// Enable using available suggestions during completion. + Future subscribeToAvailableSuggestions() async { + await server.send( + 'completion.setSubscriptions', + CompletionSetSubscriptionsParams( + [CompletionService.AVAILABLE_SUGGESTION_SETS], + ).toJson(), + ); + } + /// Enable [ServerService.STATUS] notifications so that [analysisFinished] /// can be used. Future subscribeToStatusNotifications() async {