mirror of
https://github.com/dart-lang/sdk
synced 2024-09-04 16:03:44 +00:00
[analyzer] Add benchmarks of running dart analyze
This CL: * Adds a benchmark of running `dart analyze` on a single small file. * Adds a benchmark of running `dart analyze` on a single project. * Adds a benchmark of running `dart analyze` on several projects. * Adds a hidden flag to `dart analyze` so it reports ram usage: run via `dart analyze --format=json --memory` and the memory usage will be reported in the json output. All the bencmarks run without and with cache for speed testing, and without and with cache when measuring memory usage. The idea of running this via `dart analyze` instead of running either the script or the snapshot is to measure the "real world" speed which could be different (although in practise it _does_ just run the snapshot). Future CL(s) should also add benchmarks for queries using the language server. Change-Id: Iad6d6d72c1a2ed18ab51d056b4914f8b6eb963e4 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/276100 Reviewed-by: Brian Wilkerson <brianwilkerson@google.com> Reviewed-by: Konstantin Shcheglov <scheglov@google.com> Commit-Queue: Jens Johansen <jensj@google.com>
This commit is contained in:
parent
fdeb41d111
commit
fa88e0c91f
|
@ -17,6 +17,7 @@ import 'package:args/command_runner.dart';
|
||||||
import 'package:path/path.dart' as path;
|
import 'package:path/path.dart' as path;
|
||||||
|
|
||||||
import 'perf/benchmarks_impl.dart';
|
import 'perf/benchmarks_impl.dart';
|
||||||
|
import 'perf/dart_analyze.dart';
|
||||||
import 'perf/flutter_analyze_benchmark.dart';
|
import 'perf/flutter_analyze_benchmark.dart';
|
||||||
import 'perf/flutter_completion_benchmark.dart';
|
import 'perf/flutter_completion_benchmark.dart';
|
||||||
|
|
||||||
|
@ -26,6 +27,9 @@ Future main(List<String> args) async {
|
||||||
ColdAnalysisBenchmark(ServerBenchmark.lsp),
|
ColdAnalysisBenchmark(ServerBenchmark.lsp),
|
||||||
AnalysisBenchmark(ServerBenchmark.das),
|
AnalysisBenchmark(ServerBenchmark.das),
|
||||||
AnalysisBenchmark(ServerBenchmark.lsp),
|
AnalysisBenchmark(ServerBenchmark.lsp),
|
||||||
|
CmdLineSmallFileBenchmark(),
|
||||||
|
CmdLineOneProjectBenchmark(),
|
||||||
|
CmdLineSeveralProjectsBenchmark(),
|
||||||
FlutterAnalyzeBenchmark(),
|
FlutterAnalyzeBenchmark(),
|
||||||
FlutterCompletionBenchmark.das,
|
FlutterCompletionBenchmark.das,
|
||||||
FlutterCompletionBenchmark.lsp,
|
FlutterCompletionBenchmark.lsp,
|
||||||
|
@ -92,7 +96,7 @@ abstract class Benchmark {
|
||||||
}
|
}
|
||||||
|
|
||||||
class BenchMarkResult {
|
class BenchMarkResult {
|
||||||
/// One of 'bytes', 'micros', or 'compound'.
|
/// One of 'bytes', 'kb', 'micros', or 'compound'.
|
||||||
final String kindName;
|
final String kindName;
|
||||||
|
|
||||||
final int value;
|
final int value;
|
||||||
|
|
165
pkg/analysis_server/benchmark/perf/dart_analyze.dart
Normal file
165
pkg/analysis_server/benchmark/perf/dart_analyze.dart
Normal file
|
@ -0,0 +1,165 @@
|
||||||
|
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
|
||||||
|
// for details. All rights reserved. Use of this source code is governed by a
|
||||||
|
// BSD-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
import 'package:analyzer_utilities/package_root.dart';
|
||||||
|
|
||||||
|
import '../benchmarks.dart';
|
||||||
|
import 'utils.dart';
|
||||||
|
|
||||||
|
abstract class AbtractCmdLineBenchmark extends Benchmark {
|
||||||
|
AbtractCmdLineBenchmark(super.id, super.description, {required super.kind});
|
||||||
|
|
||||||
|
@override
|
||||||
|
int get maxIterations => 3;
|
||||||
|
|
||||||
|
String get workingDir;
|
||||||
|
|
||||||
|
List<String> analyzeWhat(bool quick);
|
||||||
|
|
||||||
|
void cleanup() {}
|
||||||
|
|
||||||
|
@override
|
||||||
|
Future<BenchMarkResult> run(
|
||||||
|
{required String dartSdkPath,
|
||||||
|
bool quick = false,
|
||||||
|
bool verbose = false}) async {
|
||||||
|
if (!quick) {
|
||||||
|
deleteServerCache();
|
||||||
|
}
|
||||||
|
|
||||||
|
setup();
|
||||||
|
var analyzeThis = analyzeWhat(quick);
|
||||||
|
|
||||||
|
var stopwatchNoCache = Stopwatch()..start();
|
||||||
|
await runProcess(
|
||||||
|
'$dartSdkPath/bin/dart',
|
||||||
|
['analyze', ...analyzeThis],
|
||||||
|
cwd: workingDir,
|
||||||
|
failOnError: true,
|
||||||
|
verbose: false,
|
||||||
|
);
|
||||||
|
stopwatchNoCache.stop();
|
||||||
|
|
||||||
|
var stopwatchWithCache = Stopwatch()..start();
|
||||||
|
await runProcess(
|
||||||
|
'$dartSdkPath/bin/dart',
|
||||||
|
['analyze', ...analyzeThis],
|
||||||
|
cwd: workingDir,
|
||||||
|
failOnError: true,
|
||||||
|
verbose: false,
|
||||||
|
);
|
||||||
|
stopwatchWithCache.stop();
|
||||||
|
|
||||||
|
var result = CompoundBenchMarkResult(id);
|
||||||
|
result.add('no-cache',
|
||||||
|
BenchMarkResult('micros', stopwatchNoCache.elapsedMicroseconds));
|
||||||
|
result.add('with-cache',
|
||||||
|
BenchMarkResult('micros', stopwatchWithCache.elapsedMicroseconds));
|
||||||
|
|
||||||
|
if (!quick) {
|
||||||
|
deleteServerCache();
|
||||||
|
List<String> stdout = [];
|
||||||
|
await runProcess(
|
||||||
|
'$dartSdkPath/bin/dart',
|
||||||
|
['analyze', '--format=json', '--memory', ...analyzeThis],
|
||||||
|
cwd: workingDir,
|
||||||
|
failOnError: true,
|
||||||
|
verbose: false,
|
||||||
|
stdout: stdout,
|
||||||
|
);
|
||||||
|
int kbNoCache = jsonDecode(stdout[1])["memory"] as int;
|
||||||
|
result.add('no-cache-memory', BenchMarkResult('kb', kbNoCache));
|
||||||
|
|
||||||
|
stdout = [];
|
||||||
|
await runProcess(
|
||||||
|
'$dartSdkPath/bin/dart',
|
||||||
|
['analyze', '--format=json', '--memory', ...analyzeThis],
|
||||||
|
cwd: workingDir,
|
||||||
|
failOnError: true,
|
||||||
|
verbose: false,
|
||||||
|
stdout: stdout,
|
||||||
|
);
|
||||||
|
int kbWithCache = jsonDecode(stdout[1])["memory"] as int;
|
||||||
|
result.add('with-cache-memory', BenchMarkResult('kb', kbWithCache));
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup();
|
||||||
|
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
void setup() {}
|
||||||
|
}
|
||||||
|
|
||||||
|
class CmdLineOneProjectBenchmark extends AbtractCmdLineBenchmark {
|
||||||
|
CmdLineOneProjectBenchmark()
|
||||||
|
: super('dart-analyze-one-project',
|
||||||
|
'Run dart analyze on one project with and without a cache',
|
||||||
|
kind: 'group');
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get workingDir => packageRoot;
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<String> analyzeWhat(bool quick) =>
|
||||||
|
quick ? ["meta"] : ["analysis_server"];
|
||||||
|
}
|
||||||
|
|
||||||
|
class CmdLineSeveralProjectsBenchmark extends AbtractCmdLineBenchmark {
|
||||||
|
CmdLineSeveralProjectsBenchmark()
|
||||||
|
: super('dart-analyze-several-projects',
|
||||||
|
'Run dart analyze on several projects with and without a cache',
|
||||||
|
kind: 'group');
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get workingDir => packageRoot;
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<String> analyzeWhat(bool quick) => quick
|
||||||
|
? ["meta"]
|
||||||
|
: [
|
||||||
|
"analysis_server",
|
||||||
|
"analysis_server_client",
|
||||||
|
"analyzer",
|
||||||
|
"analyzer_cli",
|
||||||
|
"analyzer_plugin",
|
||||||
|
"analyzer_utilities",
|
||||||
|
"_fe_analyzer_shared",
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
class CmdLineSmallFileBenchmark extends AbtractCmdLineBenchmark {
|
||||||
|
Directory? _tempDir;
|
||||||
|
|
||||||
|
CmdLineSmallFileBenchmark()
|
||||||
|
: super('dart-analyze-small-file',
|
||||||
|
'Run dart analyze on a small file with and without a cache',
|
||||||
|
kind: 'group');
|
||||||
|
|
||||||
|
@override
|
||||||
|
String get workingDir => _tempDir!.path;
|
||||||
|
|
||||||
|
@override
|
||||||
|
List<String> analyzeWhat(bool quick) => ["t.dart"];
|
||||||
|
|
||||||
|
@override
|
||||||
|
void cleanup() {
|
||||||
|
_tempDir!.deleteSync(recursive: true);
|
||||||
|
_tempDir = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@override
|
||||||
|
void setup() {
|
||||||
|
var dir = Directory.systemTemp.createTempSync("analyzer-benchmark");
|
||||||
|
var file = File.fromUri(dir.uri.resolve("t.dart"));
|
||||||
|
file.writeAsStringSync("""
|
||||||
|
void main() {
|
||||||
|
print("Hello, world!");
|
||||||
|
}""");
|
||||||
|
_tempDir = dir;
|
||||||
|
}
|
||||||
|
}
|
|
@ -2,39 +2,8 @@
|
||||||
// for details. All rights reserved. Use of this source code is governed by a
|
// for details. All rights reserved. Use of this source code is governed by a
|
||||||
// BSD-style license that can be found in the LICENSE file.
|
// BSD-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
import 'dart:convert';
|
|
||||||
import 'dart:io';
|
|
||||||
|
|
||||||
import '../benchmarks.dart';
|
import '../benchmarks.dart';
|
||||||
|
import 'utils.dart';
|
||||||
Future<int> _runProcess(
|
|
||||||
String command,
|
|
||||||
List<String> args, {
|
|
||||||
String? cwd,
|
|
||||||
bool failOnError = true,
|
|
||||||
}) async {
|
|
||||||
print('\n$command ${args.join(' ')}');
|
|
||||||
|
|
||||||
var process = await Process.start(command, args, workingDirectory: cwd);
|
|
||||||
|
|
||||||
process.stdout
|
|
||||||
.transform(utf8.decoder)
|
|
||||||
.transform(LineSplitter())
|
|
||||||
.listen((line) {
|
|
||||||
print(' $line');
|
|
||||||
});
|
|
||||||
process.stderr
|
|
||||||
.transform(utf8.decoder)
|
|
||||||
.transform(LineSplitter())
|
|
||||||
.listen((line) => print(' $line'));
|
|
||||||
|
|
||||||
var exitCode = await process.exitCode;
|
|
||||||
if (exitCode != 0 && failOnError) {
|
|
||||||
throw '$command exited with $exitCode';
|
|
||||||
}
|
|
||||||
|
|
||||||
return exitCode;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// benchmarks:
|
/// benchmarks:
|
||||||
/// - analysis-flutter-analyze
|
/// - analysis-flutter-analyze
|
||||||
|
@ -65,7 +34,7 @@ class FlutterAnalyzeBenchmark extends Benchmark implements FlutterBenchmark {
|
||||||
|
|
||||||
var stopwatch = Stopwatch()..start();
|
var stopwatch = Stopwatch()..start();
|
||||||
|
|
||||||
await _runProcess(
|
await runProcess(
|
||||||
'$dartSdkPath/bin/dart',
|
'$dartSdkPath/bin/dart',
|
||||||
[
|
[
|
||||||
'packages/flutter_tools/bin/flutter_tools.dart',
|
'packages/flutter_tools/bin/flutter_tools.dart',
|
||||||
|
|
48
pkg/analysis_server/benchmark/perf/utils.dart
Normal file
48
pkg/analysis_server/benchmark/perf/utils.dart
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
|
||||||
|
// for details. All rights reserved. Use of this source code is governed by a
|
||||||
|
// BSD-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
import 'dart:convert';
|
||||||
|
import 'dart:io';
|
||||||
|
|
||||||
|
Future<int> runProcess(
|
||||||
|
String command,
|
||||||
|
List<String> args, {
|
||||||
|
String? cwd,
|
||||||
|
bool failOnError = true,
|
||||||
|
bool verbose = true,
|
||||||
|
List<String>? stdout,
|
||||||
|
}) async {
|
||||||
|
if (verbose) {
|
||||||
|
print('\n$command ${args.join(' ')}');
|
||||||
|
}
|
||||||
|
|
||||||
|
var process = await Process.start(command, args, workingDirectory: cwd);
|
||||||
|
|
||||||
|
process.stdout
|
||||||
|
.transform(utf8.decoder)
|
||||||
|
.transform(LineSplitter())
|
||||||
|
.listen((line) {
|
||||||
|
if (verbose) {
|
||||||
|
print(' $line');
|
||||||
|
}
|
||||||
|
if (stdout != null) {
|
||||||
|
stdout.add(line);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
process.stderr
|
||||||
|
.transform(utf8.decoder)
|
||||||
|
.transform(LineSplitter())
|
||||||
|
.listen((line) {
|
||||||
|
if (verbose) {
|
||||||
|
print(' $line');
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
var exitCode = await process.exitCode;
|
||||||
|
if (exitCode != 0 && failOnError) {
|
||||||
|
throw '$command exited with $exitCode';
|
||||||
|
}
|
||||||
|
|
||||||
|
return exitCode;
|
||||||
|
}
|
|
@ -94,7 +94,8 @@ class AnalysisServer {
|
||||||
|
|
||||||
final Map<String, Completer<Map<String, dynamic>>> _requestCompleters = {};
|
final Map<String, Completer<Map<String, dynamic>>> _requestCompleters = {};
|
||||||
|
|
||||||
Future<void> start({bool setAnalysisRoots = true}) async {
|
/// Starts the process and returns the pid for it.
|
||||||
|
Future<int> start({bool setAnalysisRoots = true}) async {
|
||||||
preAnalysisServerStart?.call(commandName, analysisRoots, argResults);
|
preAnalysisServerStart?.call(commandName, analysisRoots, argResults);
|
||||||
final command = [
|
final command = [
|
||||||
sdk.analysisServerSnapshot,
|
sdk.analysisServerSnapshot,
|
||||||
|
@ -186,6 +187,8 @@ class AnalysisServer {
|
||||||
'excluded': [],
|
'excluded': [],
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return process.pid;
|
||||||
}
|
}
|
||||||
|
|
||||||
Future<String> getVersion() {
|
Future<String> getVersion() {
|
||||||
|
|
|
@ -6,6 +6,7 @@ import 'dart:async';
|
||||||
import 'dart:convert';
|
import 'dart:convert';
|
||||||
import 'dart:io' as io;
|
import 'dart:io' as io;
|
||||||
|
|
||||||
|
import 'package:analysis_server/src/utilities/profiling.dart';
|
||||||
import 'package:analyzer/src/dart/analysis/experiments.dart';
|
import 'package:analyzer/src/dart/analysis/experiments.dart';
|
||||||
import 'package:cli_util/cli_logging.dart';
|
import 'package:cli_util/cli_logging.dart';
|
||||||
import 'package:meta/meta.dart';
|
import 'package:meta/meta.dart';
|
||||||
|
@ -52,6 +53,12 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
help: 'Override the location of the analysis cache.',
|
help: 'Override the location of the analysis cache.',
|
||||||
hide: !verbose,
|
hide: !verbose,
|
||||||
)
|
)
|
||||||
|
..addFlag(
|
||||||
|
'memory',
|
||||||
|
help: 'Attempt to print memory usage before exiting. '
|
||||||
|
'Will only print if format is json.',
|
||||||
|
hide: !verbose,
|
||||||
|
)
|
||||||
..addOption(
|
..addOption(
|
||||||
'format',
|
'format',
|
||||||
valueHelp: 'value',
|
valueHelp: 'value',
|
||||||
|
@ -112,6 +119,7 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
|
|
||||||
final machineFormat = args['format'] == 'machine';
|
final machineFormat = args['format'] == 'machine';
|
||||||
final jsonFormat = args['format'] == 'json';
|
final jsonFormat = args['format'] == 'json';
|
||||||
|
final printMemory = args['memory'] && jsonFormat;
|
||||||
|
|
||||||
io.Directory sdkPath;
|
io.Directory sdkPath;
|
||||||
if (args.wasParsed('sdk-path')) {
|
if (args.wasParsed('sdk-path')) {
|
||||||
|
@ -171,7 +179,7 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
error.type != 'TODO' || error.severity != 'INFO'));
|
error.type != 'TODO' || error.severity != 'INFO'));
|
||||||
});
|
});
|
||||||
|
|
||||||
await server.start();
|
int pid = await server.start();
|
||||||
|
|
||||||
bool analysisFinished = false;
|
bool analysisFinished = false;
|
||||||
|
|
||||||
|
@ -190,12 +198,20 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
await server.analysisFinished;
|
await server.analysisFinished;
|
||||||
analysisFinished = true;
|
analysisFinished = true;
|
||||||
|
|
||||||
|
UsageInfo? usageInfo;
|
||||||
|
if (printMemory) {
|
||||||
|
usageInfo =
|
||||||
|
await ProcessProfiler.getProfilerForPlatform()?.getProcessUsage(pid);
|
||||||
|
}
|
||||||
|
|
||||||
await server.shutdown();
|
await server.shutdown();
|
||||||
|
|
||||||
progress?.finish(showTiming: true);
|
progress?.finish(showTiming: true);
|
||||||
|
|
||||||
if (errors.isEmpty) {
|
if (errors.isEmpty) {
|
||||||
if (!machineFormat) {
|
if (printMemory && usageInfo != null) {
|
||||||
|
emitJsonFormat(log, errors, usageInfo);
|
||||||
|
} else if (!machineFormat) {
|
||||||
log.stdout('No issues found!');
|
log.stdout('No issues found!');
|
||||||
}
|
}
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -206,7 +222,7 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
if (machineFormat) {
|
if (machineFormat) {
|
||||||
emitMachineFormat(log, errors);
|
emitMachineFormat(log, errors);
|
||||||
} else if (jsonFormat) {
|
} else if (jsonFormat) {
|
||||||
emitJsonFormat(log, errors);
|
emitJsonFormat(log, errors, usageInfo);
|
||||||
} else {
|
} else {
|
||||||
var relativeTo = targets.length == 1 ? targets.single : null;
|
var relativeTo = targets.length == 1 ? targets.single : null;
|
||||||
|
|
||||||
|
@ -325,7 +341,8 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
}
|
}
|
||||||
|
|
||||||
@visibleForTesting
|
@visibleForTesting
|
||||||
static void emitJsonFormat(Logger log, List<AnalysisError> errors) {
|
static void emitJsonFormat(
|
||||||
|
Logger log, List<AnalysisError> errors, UsageInfo? usageInfo) {
|
||||||
Map<String, dynamic> location(
|
Map<String, dynamic> location(
|
||||||
String filePath, Map<String, dynamic> range) =>
|
String filePath, Map<String, dynamic> range) =>
|
||||||
{
|
{
|
||||||
|
@ -382,6 +399,7 @@ class AnalyzeCommand extends DartdevCommand {
|
||||||
log.stdout(json.encode({
|
log.stdout(json.encode({
|
||||||
'version': 1,
|
'version': 1,
|
||||||
'diagnostics': diagnostics,
|
'diagnostics': diagnostics,
|
||||||
|
if (usageInfo != null) 'memory': usageInfo.memoryKB
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -580,7 +580,7 @@ void f() {
|
||||||
final logger = TestLogger(false);
|
final logger = TestLogger(false);
|
||||||
final errors = [AnalysisError(sampleInfoJson)];
|
final errors = [AnalysisError(sampleInfoJson)];
|
||||||
|
|
||||||
AnalyzeCommand.emitJsonFormat(logger, errors);
|
AnalyzeCommand.emitJsonFormat(logger, errors, null);
|
||||||
|
|
||||||
expect(logger.stderrBuffer, isEmpty);
|
expect(logger.stderrBuffer, isEmpty);
|
||||||
final stdout = logger.stdoutBuffer.toString().trim();
|
final stdout = logger.stdoutBuffer.toString().trim();
|
||||||
|
@ -595,7 +595,7 @@ void f() {
|
||||||
final logger = TestLogger(false);
|
final logger = TestLogger(false);
|
||||||
final errors = [AnalysisError(fullDiagnosticJson)];
|
final errors = [AnalysisError(fullDiagnosticJson)];
|
||||||
|
|
||||||
AnalyzeCommand.emitJsonFormat(logger, errors);
|
AnalyzeCommand.emitJsonFormat(logger, errors, null);
|
||||||
|
|
||||||
expect(logger.stderrBuffer, isEmpty);
|
expect(logger.stderrBuffer, isEmpty);
|
||||||
final stdout = logger.stdoutBuffer.toString().trim();
|
final stdout = logger.stdoutBuffer.toString().trim();
|
||||||
|
|
Loading…
Reference in a new issue