From fa88e0c91fa54f99305e6dddffac0400c0801d67 Mon Sep 17 00:00:00 2001 From: Jens Johansen Date: Mon, 19 Dec 2022 08:22:32 +0000 Subject: [PATCH] [analyzer] Add benchmarks of running `dart analyze` This CL: * Adds a benchmark of running `dart analyze` on a single small file. * Adds a benchmark of running `dart analyze` on a single project. * Adds a benchmark of running `dart analyze` on several projects. * Adds a hidden flag to `dart analyze` so it reports ram usage: run via `dart analyze --format=json --memory` and the memory usage will be reported in the json output. All the bencmarks run without and with cache for speed testing, and without and with cache when measuring memory usage. The idea of running this via `dart analyze` instead of running either the script or the snapshot is to measure the "real world" speed which could be different (although in practise it _does_ just run the snapshot). Future CL(s) should also add benchmarks for queries using the language server. Change-Id: Iad6d6d72c1a2ed18ab51d056b4914f8b6eb963e4 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/276100 Reviewed-by: Brian Wilkerson Reviewed-by: Konstantin Shcheglov Commit-Queue: Jens Johansen --- pkg/analysis_server/benchmark/benchmarks.dart | 6 +- .../benchmark/perf/dart_analyze.dart | 165 ++++++++++++++++++ .../perf/flutter_analyze_benchmark.dart | 35 +--- pkg/analysis_server/benchmark/perf/utils.dart | 48 +++++ pkg/dartdev/lib/src/analysis_server.dart | 5 +- pkg/dartdev/lib/src/commands/analyze.dart | 26 ++- pkg/dartdev/test/commands/analyze_test.dart | 4 +- 7 files changed, 248 insertions(+), 41 deletions(-) create mode 100644 pkg/analysis_server/benchmark/perf/dart_analyze.dart create mode 100644 pkg/analysis_server/benchmark/perf/utils.dart diff --git a/pkg/analysis_server/benchmark/benchmarks.dart b/pkg/analysis_server/benchmark/benchmarks.dart index 6ed32465f01..704dc47f486 100644 --- a/pkg/analysis_server/benchmark/benchmarks.dart +++ b/pkg/analysis_server/benchmark/benchmarks.dart @@ -17,6 +17,7 @@ import 'package:args/command_runner.dart'; import 'package:path/path.dart' as path; import 'perf/benchmarks_impl.dart'; +import 'perf/dart_analyze.dart'; import 'perf/flutter_analyze_benchmark.dart'; import 'perf/flutter_completion_benchmark.dart'; @@ -26,6 +27,9 @@ Future main(List args) async { ColdAnalysisBenchmark(ServerBenchmark.lsp), AnalysisBenchmark(ServerBenchmark.das), AnalysisBenchmark(ServerBenchmark.lsp), + CmdLineSmallFileBenchmark(), + CmdLineOneProjectBenchmark(), + CmdLineSeveralProjectsBenchmark(), FlutterAnalyzeBenchmark(), FlutterCompletionBenchmark.das, FlutterCompletionBenchmark.lsp, @@ -92,7 +96,7 @@ abstract class Benchmark { } class BenchMarkResult { - /// One of 'bytes', 'micros', or 'compound'. + /// One of 'bytes', 'kb', 'micros', or 'compound'. final String kindName; final int value; diff --git a/pkg/analysis_server/benchmark/perf/dart_analyze.dart b/pkg/analysis_server/benchmark/perf/dart_analyze.dart new file mode 100644 index 00000000000..0a2b7f4d0c8 --- /dev/null +++ b/pkg/analysis_server/benchmark/perf/dart_analyze.dart @@ -0,0 +1,165 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:analyzer_utilities/package_root.dart'; + +import '../benchmarks.dart'; +import 'utils.dart'; + +abstract class AbtractCmdLineBenchmark extends Benchmark { + AbtractCmdLineBenchmark(super.id, super.description, {required super.kind}); + + @override + int get maxIterations => 3; + + String get workingDir; + + List analyzeWhat(bool quick); + + void cleanup() {} + + @override + Future run( + {required String dartSdkPath, + bool quick = false, + bool verbose = false}) async { + if (!quick) { + deleteServerCache(); + } + + setup(); + var analyzeThis = analyzeWhat(quick); + + var stopwatchNoCache = Stopwatch()..start(); + await runProcess( + '$dartSdkPath/bin/dart', + ['analyze', ...analyzeThis], + cwd: workingDir, + failOnError: true, + verbose: false, + ); + stopwatchNoCache.stop(); + + var stopwatchWithCache = Stopwatch()..start(); + await runProcess( + '$dartSdkPath/bin/dart', + ['analyze', ...analyzeThis], + cwd: workingDir, + failOnError: true, + verbose: false, + ); + stopwatchWithCache.stop(); + + var result = CompoundBenchMarkResult(id); + result.add('no-cache', + BenchMarkResult('micros', stopwatchNoCache.elapsedMicroseconds)); + result.add('with-cache', + BenchMarkResult('micros', stopwatchWithCache.elapsedMicroseconds)); + + if (!quick) { + deleteServerCache(); + List stdout = []; + await runProcess( + '$dartSdkPath/bin/dart', + ['analyze', '--format=json', '--memory', ...analyzeThis], + cwd: workingDir, + failOnError: true, + verbose: false, + stdout: stdout, + ); + int kbNoCache = jsonDecode(stdout[1])["memory"] as int; + result.add('no-cache-memory', BenchMarkResult('kb', kbNoCache)); + + stdout = []; + await runProcess( + '$dartSdkPath/bin/dart', + ['analyze', '--format=json', '--memory', ...analyzeThis], + cwd: workingDir, + failOnError: true, + verbose: false, + stdout: stdout, + ); + int kbWithCache = jsonDecode(stdout[1])["memory"] as int; + result.add('with-cache-memory', BenchMarkResult('kb', kbWithCache)); + } + + cleanup(); + + return result; + } + + void setup() {} +} + +class CmdLineOneProjectBenchmark extends AbtractCmdLineBenchmark { + CmdLineOneProjectBenchmark() + : super('dart-analyze-one-project', + 'Run dart analyze on one project with and without a cache', + kind: 'group'); + + @override + String get workingDir => packageRoot; + + @override + List analyzeWhat(bool quick) => + quick ? ["meta"] : ["analysis_server"]; +} + +class CmdLineSeveralProjectsBenchmark extends AbtractCmdLineBenchmark { + CmdLineSeveralProjectsBenchmark() + : super('dart-analyze-several-projects', + 'Run dart analyze on several projects with and without a cache', + kind: 'group'); + + @override + String get workingDir => packageRoot; + + @override + List analyzeWhat(bool quick) => quick + ? ["meta"] + : [ + "analysis_server", + "analysis_server_client", + "analyzer", + "analyzer_cli", + "analyzer_plugin", + "analyzer_utilities", + "_fe_analyzer_shared", + ]; +} + +class CmdLineSmallFileBenchmark extends AbtractCmdLineBenchmark { + Directory? _tempDir; + + CmdLineSmallFileBenchmark() + : super('dart-analyze-small-file', + 'Run dart analyze on a small file with and without a cache', + kind: 'group'); + + @override + String get workingDir => _tempDir!.path; + + @override + List analyzeWhat(bool quick) => ["t.dart"]; + + @override + void cleanup() { + _tempDir!.deleteSync(recursive: true); + _tempDir = null; + } + + @override + void setup() { + var dir = Directory.systemTemp.createTempSync("analyzer-benchmark"); + var file = File.fromUri(dir.uri.resolve("t.dart")); + file.writeAsStringSync(""" +void main() { + print("Hello, world!"); +}"""); + _tempDir = dir; + } +} diff --git a/pkg/analysis_server/benchmark/perf/flutter_analyze_benchmark.dart b/pkg/analysis_server/benchmark/perf/flutter_analyze_benchmark.dart index 3c4ae7741ac..f5bf2d5db1a 100644 --- a/pkg/analysis_server/benchmark/perf/flutter_analyze_benchmark.dart +++ b/pkg/analysis_server/benchmark/perf/flutter_analyze_benchmark.dart @@ -2,39 +2,8 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -import 'dart:convert'; -import 'dart:io'; - import '../benchmarks.dart'; - -Future _runProcess( - String command, - List args, { - String? cwd, - bool failOnError = true, -}) async { - print('\n$command ${args.join(' ')}'); - - var process = await Process.start(command, args, workingDirectory: cwd); - - process.stdout - .transform(utf8.decoder) - .transform(LineSplitter()) - .listen((line) { - print(' $line'); - }); - process.stderr - .transform(utf8.decoder) - .transform(LineSplitter()) - .listen((line) => print(' $line')); - - var exitCode = await process.exitCode; - if (exitCode != 0 && failOnError) { - throw '$command exited with $exitCode'; - } - - return exitCode; -} +import 'utils.dart'; /// benchmarks: /// - analysis-flutter-analyze @@ -65,7 +34,7 @@ class FlutterAnalyzeBenchmark extends Benchmark implements FlutterBenchmark { var stopwatch = Stopwatch()..start(); - await _runProcess( + await runProcess( '$dartSdkPath/bin/dart', [ 'packages/flutter_tools/bin/flutter_tools.dart', diff --git a/pkg/analysis_server/benchmark/perf/utils.dart b/pkg/analysis_server/benchmark/perf/utils.dart new file mode 100644 index 00000000000..a17291246ce --- /dev/null +++ b/pkg/analysis_server/benchmark/perf/utils.dart @@ -0,0 +1,48 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +Future runProcess( + String command, + List args, { + String? cwd, + bool failOnError = true, + bool verbose = true, + List? stdout, +}) async { + if (verbose) { + print('\n$command ${args.join(' ')}'); + } + + var process = await Process.start(command, args, workingDirectory: cwd); + + process.stdout + .transform(utf8.decoder) + .transform(LineSplitter()) + .listen((line) { + if (verbose) { + print(' $line'); + } + if (stdout != null) { + stdout.add(line); + } + }); + process.stderr + .transform(utf8.decoder) + .transform(LineSplitter()) + .listen((line) { + if (verbose) { + print(' $line'); + } + }); + + var exitCode = await process.exitCode; + if (exitCode != 0 && failOnError) { + throw '$command exited with $exitCode'; + } + + return exitCode; +} diff --git a/pkg/dartdev/lib/src/analysis_server.dart b/pkg/dartdev/lib/src/analysis_server.dart index 3b89cd191a7..44af80a08d9 100644 --- a/pkg/dartdev/lib/src/analysis_server.dart +++ b/pkg/dartdev/lib/src/analysis_server.dart @@ -94,7 +94,8 @@ class AnalysisServer { final Map>> _requestCompleters = {}; - Future start({bool setAnalysisRoots = true}) async { + /// Starts the process and returns the pid for it. + Future start({bool setAnalysisRoots = true}) async { preAnalysisServerStart?.call(commandName, analysisRoots, argResults); final command = [ sdk.analysisServerSnapshot, @@ -186,6 +187,8 @@ class AnalysisServer { 'excluded': [], }); } + + return process.pid; } Future getVersion() { diff --git a/pkg/dartdev/lib/src/commands/analyze.dart b/pkg/dartdev/lib/src/commands/analyze.dart index 047e1c7d11e..7eab8135137 100644 --- a/pkg/dartdev/lib/src/commands/analyze.dart +++ b/pkg/dartdev/lib/src/commands/analyze.dart @@ -6,6 +6,7 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io' as io; +import 'package:analysis_server/src/utilities/profiling.dart'; import 'package:analyzer/src/dart/analysis/experiments.dart'; import 'package:cli_util/cli_logging.dart'; import 'package:meta/meta.dart'; @@ -52,6 +53,12 @@ class AnalyzeCommand extends DartdevCommand { help: 'Override the location of the analysis cache.', hide: !verbose, ) + ..addFlag( + 'memory', + help: 'Attempt to print memory usage before exiting. ' + 'Will only print if format is json.', + hide: !verbose, + ) ..addOption( 'format', valueHelp: 'value', @@ -112,6 +119,7 @@ class AnalyzeCommand extends DartdevCommand { final machineFormat = args['format'] == 'machine'; final jsonFormat = args['format'] == 'json'; + final printMemory = args['memory'] && jsonFormat; io.Directory sdkPath; if (args.wasParsed('sdk-path')) { @@ -171,7 +179,7 @@ class AnalyzeCommand extends DartdevCommand { error.type != 'TODO' || error.severity != 'INFO')); }); - await server.start(); + int pid = await server.start(); bool analysisFinished = false; @@ -190,12 +198,20 @@ class AnalyzeCommand extends DartdevCommand { await server.analysisFinished; analysisFinished = true; + UsageInfo? usageInfo; + if (printMemory) { + usageInfo = + await ProcessProfiler.getProfilerForPlatform()?.getProcessUsage(pid); + } + await server.shutdown(); progress?.finish(showTiming: true); if (errors.isEmpty) { - if (!machineFormat) { + if (printMemory && usageInfo != null) { + emitJsonFormat(log, errors, usageInfo); + } else if (!machineFormat) { log.stdout('No issues found!'); } return 0; @@ -206,7 +222,7 @@ class AnalyzeCommand extends DartdevCommand { if (machineFormat) { emitMachineFormat(log, errors); } else if (jsonFormat) { - emitJsonFormat(log, errors); + emitJsonFormat(log, errors, usageInfo); } else { var relativeTo = targets.length == 1 ? targets.single : null; @@ -325,7 +341,8 @@ class AnalyzeCommand extends DartdevCommand { } @visibleForTesting - static void emitJsonFormat(Logger log, List errors) { + static void emitJsonFormat( + Logger log, List errors, UsageInfo? usageInfo) { Map location( String filePath, Map range) => { @@ -382,6 +399,7 @@ class AnalyzeCommand extends DartdevCommand { log.stdout(json.encode({ 'version': 1, 'diagnostics': diagnostics, + if (usageInfo != null) 'memory': usageInfo.memoryKB })); } diff --git a/pkg/dartdev/test/commands/analyze_test.dart b/pkg/dartdev/test/commands/analyze_test.dart index 1750ad6098a..686aea07430 100644 --- a/pkg/dartdev/test/commands/analyze_test.dart +++ b/pkg/dartdev/test/commands/analyze_test.dart @@ -580,7 +580,7 @@ void f() { final logger = TestLogger(false); final errors = [AnalysisError(sampleInfoJson)]; - AnalyzeCommand.emitJsonFormat(logger, errors); + AnalyzeCommand.emitJsonFormat(logger, errors, null); expect(logger.stderrBuffer, isEmpty); final stdout = logger.stdoutBuffer.toString().trim(); @@ -595,7 +595,7 @@ void f() { final logger = TestLogger(false); final errors = [AnalysisError(fullDiagnosticJson)]; - AnalyzeCommand.emitJsonFormat(logger, errors); + AnalyzeCommand.emitJsonFormat(logger, errors, null); expect(logger.stderrBuffer, isEmpty); final stdout = logger.stdoutBuffer.toString().trim();