[analyzer] Add benchmarks of running dart analyze

This CL:
  * Adds a benchmark of running `dart analyze` on a single small file.
  * Adds a benchmark of running `dart analyze` on a single project.
  * Adds a benchmark of running `dart analyze` on several projects.
  * Adds a hidden flag to `dart analyze` so it reports ram usage:
    run via `dart analyze --format=json --memory` and the memory
    usage will be reported in the json output.

All the bencmarks run without and with cache for speed testing, and
without and with cache when measuring memory usage.

The idea of running this via `dart analyze` instead of running either
the script or the snapshot is to measure the "real world" speed which
could be different (although in practise it _does_ just run the
snapshot).

Future CL(s) should also add benchmarks for queries using the
language server.

Change-Id: Iad6d6d72c1a2ed18ab51d056b4914f8b6eb963e4
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/276100
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>

pkg/analysis_server/benchmark/benchmarks.dart

@@ -17,6 +17,7 @@ import 'package:args/command_runner.dart';
 import 'package:path/path.dart' as path;
 
 import 'perf/benchmarks_impl.dart';
+import 'perf/dart_analyze.dart';
 import 'perf/flutter_analyze_benchmark.dart';
 import 'perf/flutter_completion_benchmark.dart';
 
@@ -26,6 +27,9 @@ Future main(List<String> args) async {
     ColdAnalysisBenchmark(ServerBenchmark.lsp),
     AnalysisBenchmark(ServerBenchmark.das),
     AnalysisBenchmark(ServerBenchmark.lsp),
+    CmdLineSmallFileBenchmark(),
+    CmdLineOneProjectBenchmark(),
+    CmdLineSeveralProjectsBenchmark(),
     FlutterAnalyzeBenchmark(),
     FlutterCompletionBenchmark.das,
     FlutterCompletionBenchmark.lsp,
@@ -92,7 +96,7 @@ abstract class Benchmark {
 }
 
 class BenchMarkResult {
-  /// One of 'bytes', 'micros', or 'compound'.
+  /// One of 'bytes', 'kb', 'micros', or 'compound'.
   final String kindName;
 
   final int value;

pkg/analysis_server/benchmark/perf/dart_analyze.dart

@@ -0,0 +1,165 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:analyzer_utilities/package_root.dart';
+
+import '../benchmarks.dart';
+import 'utils.dart';
+
+abstract class AbtractCmdLineBenchmark extends Benchmark {
+  AbtractCmdLineBenchmark(super.id, super.description, {required super.kind});
+
+  @override
+  int get maxIterations => 3;
+
+  String get workingDir;
+
+  List<String> analyzeWhat(bool quick);
+
+  void cleanup() {}
+
+  @override
+  Future<BenchMarkResult> run(
+      {required String dartSdkPath,
+      bool quick = false,
+      bool verbose = false}) async {
+    if (!quick) {
+      deleteServerCache();
+    }
+
+    setup();
+    var analyzeThis = analyzeWhat(quick);
+
+    var stopwatchNoCache = Stopwatch()..start();
+    await runProcess(
+      '$dartSdkPath/bin/dart',
+      ['analyze', ...analyzeThis],
+      cwd: workingDir,
+      failOnError: true,
+      verbose: false,
+    );
+    stopwatchNoCache.stop();
+
+    var stopwatchWithCache = Stopwatch()..start();
+    await runProcess(
+      '$dartSdkPath/bin/dart',
+      ['analyze', ...analyzeThis],
+      cwd: workingDir,
+      failOnError: true,
+      verbose: false,
+    );
+    stopwatchWithCache.stop();
+
+    var result = CompoundBenchMarkResult(id);
+    result.add('no-cache',
+        BenchMarkResult('micros', stopwatchNoCache.elapsedMicroseconds));
+    result.add('with-cache',
+        BenchMarkResult('micros', stopwatchWithCache.elapsedMicroseconds));
+
+    if (!quick) {
+      deleteServerCache();
+      List<String> stdout = [];
+      await runProcess(
+        '$dartSdkPath/bin/dart',
+        ['analyze', '--format=json', '--memory', ...analyzeThis],
+        cwd: workingDir,
+        failOnError: true,
+        verbose: false,
+        stdout: stdout,
+      );
+      int kbNoCache = jsonDecode(stdout[1])["memory"] as int;
+      result.add('no-cache-memory', BenchMarkResult('kb', kbNoCache));
+
+      stdout = [];
+      await runProcess(
+        '$dartSdkPath/bin/dart',
+        ['analyze', '--format=json', '--memory', ...analyzeThis],
+        cwd: workingDir,
+        failOnError: true,
+        verbose: false,
+        stdout: stdout,
+      );
+      int kbWithCache = jsonDecode(stdout[1])["memory"] as int;
+      result.add('with-cache-memory', BenchMarkResult('kb', kbWithCache));
+    }
+
+    cleanup();
+
+    return result;
+  }
+
+  void setup() {}
+}
+
+class CmdLineOneProjectBenchmark extends AbtractCmdLineBenchmark {
+  CmdLineOneProjectBenchmark()
+      : super('dart-analyze-one-project',
+            'Run dart analyze on one project with and without a cache',
+            kind: 'group');
+
+  @override
+  String get workingDir => packageRoot;
+
+  @override
+  List<String> analyzeWhat(bool quick) =>
+      quick ? ["meta"] : ["analysis_server"];
+}
+
+class CmdLineSeveralProjectsBenchmark extends AbtractCmdLineBenchmark {
+  CmdLineSeveralProjectsBenchmark()
+      : super('dart-analyze-several-projects',
+            'Run dart analyze on several projects with and without a cache',
+            kind: 'group');
+
+  @override
+  String get workingDir => packageRoot;
+
+  @override
+  List<String> analyzeWhat(bool quick) => quick
+      ? ["meta"]
+      : [
+          "analysis_server",
+          "analysis_server_client",
+          "analyzer",
+          "analyzer_cli",
+          "analyzer_plugin",
+          "analyzer_utilities",
+          "_fe_analyzer_shared",
+        ];
+}
+
+class CmdLineSmallFileBenchmark extends AbtractCmdLineBenchmark {
+  Directory? _tempDir;
+
+  CmdLineSmallFileBenchmark()
+      : super('dart-analyze-small-file',
+            'Run dart analyze on a small file with and without a cache',
+            kind: 'group');
+
+  @override
+  String get workingDir => _tempDir!.path;
+
+  @override
+  List<String> analyzeWhat(bool quick) => ["t.dart"];
+
+  @override
+  void cleanup() {
+    _tempDir!.deleteSync(recursive: true);
+    _tempDir = null;
+  }
+
+  @override
+  void setup() {
+    var dir = Directory.systemTemp.createTempSync("analyzer-benchmark");
+    var file = File.fromUri(dir.uri.resolve("t.dart"));
+    file.writeAsStringSync("""
+void main() {
+  print("Hello, world!");
+}""");
+    _tempDir = dir;
+  }
+}

pkg/analysis_server/benchmark/perf/flutter_analyze_benchmark.dart

@@ -2,39 +2,8 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:convert';
-import 'dart:io';
-
 import '../benchmarks.dart';
-
-Future<int> _runProcess(
-  String command,
-  List<String> args, {
-  String? cwd,
-  bool failOnError = true,
-}) async {
-  print('\n$command ${args.join(' ')}');
-
-  var process = await Process.start(command, args, workingDirectory: cwd);
-
-  process.stdout
-      .transform(utf8.decoder)
-      .transform(LineSplitter())
-      .listen((line) {
-    print('  $line');
-  });
-  process.stderr
-      .transform(utf8.decoder)
-      .transform(LineSplitter())
-      .listen((line) => print('  $line'));
-
-  var exitCode = await process.exitCode;
-  if (exitCode != 0 && failOnError) {
-    throw '$command exited with $exitCode';
-  }
-
-  return exitCode;
-}
+import 'utils.dart';
 
 /// benchmarks:
 ///   - analysis-flutter-analyze
@@ -65,7 +34,7 @@ class FlutterAnalyzeBenchmark extends Benchmark implements FlutterBenchmark {
 
     var stopwatch = Stopwatch()..start();
 
-    await _runProcess(
+    await runProcess(
       '$dartSdkPath/bin/dart',
       [
         'packages/flutter_tools/bin/flutter_tools.dart',

pkg/analysis_server/benchmark/perf/utils.dart

@@ -0,0 +1,48 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+Future<int> runProcess(
+  String command,
+  List<String> args, {
+  String? cwd,
+  bool failOnError = true,
+  bool verbose = true,
+  List<String>? stdout,
+}) async {
+  if (verbose) {
+    print('\n$command ${args.join(' ')}');
+  }
+
+  var process = await Process.start(command, args, workingDirectory: cwd);
+
+  process.stdout
+      .transform(utf8.decoder)
+      .transform(LineSplitter())
+      .listen((line) {
+    if (verbose) {
+      print('  $line');
+    }
+    if (stdout != null) {
+      stdout.add(line);
+    }
+  });
+  process.stderr
+      .transform(utf8.decoder)
+      .transform(LineSplitter())
+      .listen((line) {
+    if (verbose) {
+      print('  $line');
+    }
+  });
+
+  var exitCode = await process.exitCode;
+  if (exitCode != 0 && failOnError) {
+    throw '$command exited with $exitCode';
+  }
+
+  return exitCode;
+}

pkg/dartdev/lib/src/analysis_server.dart

@@ -94,7 +94,8 @@ class AnalysisServer {
 
   final Map<String, Completer<Map<String, dynamic>>> _requestCompleters = {};
 
-  Future<void> start({bool setAnalysisRoots = true}) async {
+  /// Starts the process and returns the pid for it.
+  Future<int> start({bool setAnalysisRoots = true}) async {
     preAnalysisServerStart?.call(commandName, analysisRoots, argResults);
     final command = [
       sdk.analysisServerSnapshot,
@@ -186,6 +187,8 @@ class AnalysisServer {
         'excluded': [],
       });
     }
+
+    return process.pid;
   }
 
   Future<String> getVersion() {

pkg/dartdev/lib/src/commands/analyze.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 import 'dart:convert';
 import 'dart:io' as io;
 
+import 'package:analysis_server/src/utilities/profiling.dart';
 import 'package:analyzer/src/dart/analysis/experiments.dart';
 import 'package:cli_util/cli_logging.dart';
 import 'package:meta/meta.dart';
@@ -52,6 +53,12 @@ class AnalyzeCommand extends DartdevCommand {
         help: 'Override the location of the analysis cache.',
         hide: !verbose,
       )
+      ..addFlag(
+        'memory',
+        help: 'Attempt to print memory usage before exiting. '
+            'Will only print if format is json.',
+        hide: !verbose,
+      )
       ..addOption(
         'format',
         valueHelp: 'value',
@@ -112,6 +119,7 @@ class AnalyzeCommand extends DartdevCommand {
 
     final machineFormat = args['format'] == 'machine';
     final jsonFormat = args['format'] == 'json';
+    final printMemory = args['memory'] && jsonFormat;
 
     io.Directory sdkPath;
     if (args.wasParsed('sdk-path')) {
@@ -171,7 +179,7 @@ class AnalyzeCommand extends DartdevCommand {
           error.type != 'TODO' || error.severity != 'INFO'));
     });
 
-    await server.start();
+    int pid = await server.start();
 
     bool analysisFinished = false;
 
@@ -190,12 +198,20 @@ class AnalyzeCommand extends DartdevCommand {
     await server.analysisFinished;
     analysisFinished = true;
 
+    UsageInfo? usageInfo;
+    if (printMemory) {
+      usageInfo =
+          await ProcessProfiler.getProfilerForPlatform()?.getProcessUsage(pid);
+    }
+
     await server.shutdown();
 
     progress?.finish(showTiming: true);
 
     if (errors.isEmpty) {
-      if (!machineFormat) {
+      if (printMemory && usageInfo != null) {
+        emitJsonFormat(log, errors, usageInfo);
+      } else if (!machineFormat) {
         log.stdout('No issues found!');
       }
       return 0;
@@ -206,7 +222,7 @@ class AnalyzeCommand extends DartdevCommand {
     if (machineFormat) {
       emitMachineFormat(log, errors);
     } else if (jsonFormat) {
-      emitJsonFormat(log, errors);
+      emitJsonFormat(log, errors, usageInfo);
     } else {
       var relativeTo = targets.length == 1 ? targets.single : null;
 
@@ -325,7 +341,8 @@ class AnalyzeCommand extends DartdevCommand {
   }
 
   @visibleForTesting
-  static void emitJsonFormat(Logger log, List<AnalysisError> errors) {
+  static void emitJsonFormat(
+      Logger log, List<AnalysisError> errors, UsageInfo? usageInfo) {
     Map<String, dynamic> location(
             String filePath, Map<String, dynamic> range) =>
         {
@@ -382,6 +399,7 @@ class AnalyzeCommand extends DartdevCommand {
     log.stdout(json.encode({
       'version': 1,
       'diagnostics': diagnostics,
+      if (usageInfo != null) 'memory': usageInfo.memoryKB
     }));
   }
 

pkg/dartdev/test/commands/analyze_test.dart

@@ -580,7 +580,7 @@ void f() {
         final logger = TestLogger(false);
         final errors = [AnalysisError(sampleInfoJson)];
 
-        AnalyzeCommand.emitJsonFormat(logger, errors);
+        AnalyzeCommand.emitJsonFormat(logger, errors, null);
 
         expect(logger.stderrBuffer, isEmpty);
         final stdout = logger.stdoutBuffer.toString().trim();
@@ -595,7 +595,7 @@ void f() {
         final logger = TestLogger(false);
         final errors = [AnalysisError(fullDiagnosticJson)];
 
-        AnalyzeCommand.emitJsonFormat(logger, errors);
+        AnalyzeCommand.emitJsonFormat(logger, errors, null);
 
         expect(logger.stderrBuffer, isEmpty);
         final stdout = logger.stdoutBuffer.toString().trim();