Adds benchmark measuring memory consumption of live objects after analyzing flutter.

Will warm up the memory cache and then measure how much live memory the analyzer consumes after visiting all elements with the warmed up cache.

All results
--------------------------------
flutter_elements
  reachableObjects
    count: 9071480
    size: 799672175 = 780929 KB = 762 MB
  _SimpleUri
    count: 3112
    size(shallow): 248960 = 243 KB
    duplicateCount: 0

Change-Id: I75209f88f6f615172127cd439afe9d0cd83b1c4f
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/329682
Reviewed-by: Martin Kustermann <kustermann@google.com>
Commit-Queue: Konstantin Shcheglov <scheglov@google.com>

pkg/analyzer/pubspec.yaml

@@ -29,14 +29,20 @@ dev_dependencies:
   analyzer_utilities: any
   args: any
   async: any
+  heap_snapshot: any
   linter: any
   lints: any
   matcher: any
   test: any
   test_reflective_loader: any
+  vm_service: any
 
 dependency_overrides:
   _fe_analyzer_shared:
     path: ../_fe_analyzer_shared
   analyzer_utilities:
     path: ../analyzer_utilities
+  heap_snapshot:
+    path: ../heap_snapshot
+  vm_service:
+    path: ../vm_service

pkg/analyzer/tool/benchmark/heap/flutter_elements.dart

@@ -0,0 +1,316 @@
+// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:developer' as developer;
+import 'dart:io' as io;
+import 'dart:typed_data';
+
+import 'package:analyzer/dart/analysis/results.dart';
+import 'package:analyzer/dart/element/element.dart';
+import 'package:analyzer/dart/element/visitor.dart';
+import 'package:analyzer/error/error.dart';
+import 'package:analyzer/file_system/physical_file_system.dart';
+import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart';
+import 'package:analyzer/src/dart/analysis/byte_store.dart';
+import 'package:analyzer/src/dart/analysis/file_content_cache.dart';
+import 'package:analyzer/src/dart/analysis/unlinked_unit_store.dart';
+import 'package:heap_snapshot/analysis.dart';
+import 'package:heap_snapshot/format.dart';
+import 'package:vm_service/vm_service.dart';
+
+import '../../../test/util/tree_string_sink.dart';
+import 'result.dart';
+
+void main() async {
+  final byteStore = MemoryByteStore();
+
+  print('First pass, fill ByteStore');
+  await _withNewAnalysisContext<void>(
+    byteStore: byteStore,
+    (collection) async {
+      print('  Analysis contexts: ${collection.contexts.length}');
+
+      timer.start();
+      await _analyzeFiles(collection);
+      print('  [+${timer.elapsedMilliseconds} ms] Analyze');
+
+      timer.reset();
+      await _getAvailableLibraries(collection);
+      print('  [+${timer.elapsedMilliseconds} ms] Get available libraries');
+      print('');
+    },
+  );
+
+  timer.reset();
+  print('Second pass, read elements');
+  final heapBytes = await _withNewAnalysisContext(
+    byteStore: byteStore,
+    (collection) async {
+      print('  Analysis contexts: ${collection.contexts.length}');
+
+      await _analyzeFiles(collection);
+      print('  [+${timer.elapsedMilliseconds} ms] Analyze');
+
+      timer.reset();
+      await _getAvailableLibraries(collection);
+      print('  [+${timer.elapsedMilliseconds} ms] Get available libraries');
+      print('');
+
+      return _getHeapSnapshot();
+    },
+  );
+
+  final allResults = _analyzeSnapshot(heapBytes);
+
+  {
+    final buffer = StringBuffer();
+    final sink = TreeStringSink(sink: buffer, indent: '');
+    writeBenchmarkResult(sink, allResults);
+    print('All results');
+    print('-' * 32);
+    print(buffer);
+  }
+}
+
+const String includedPath = '/Users/scheglov/dart/flutter_multi/packages';
+
+final Stopwatch timer = Stopwatch();
+
+/// Analyzes all included files.
+///
+/// Throws if there is a compile-time error.
+Future<void> _analyzeFiles(
+  AnalysisContextCollectionImpl collection,
+) async {
+  for (final analysisContext in collection.contexts) {
+    final analyzedFiles = analysisContext.contextRoot.analyzedFiles().toList();
+    for (final filePath in analyzedFiles) {
+      if (filePath.endsWith('.dart')) {
+        final analysisSession = analysisContext.currentSession;
+        await analysisSession.getUnitElement(filePath);
+
+        // Check that there are no compile-time errors.
+        // We want to be sure that we get elements models.
+        final errorsResult = await analysisSession.getErrors(filePath);
+        if (errorsResult is ErrorsResult) {
+          final errors = errorsResult.errors
+              .where((element) =>
+                  element.errorCode.type == ErrorType.COMPILE_TIME_ERROR)
+              .toList();
+          if (errors.isNotEmpty) {
+            throw StateError('Errors in $filePath\n$errors');
+          }
+        }
+      }
+    }
+  }
+}
+
+BenchmarkResultCompound _analyzeSnapshot(Uint8List bytes) {
+  final allResults = BenchmarkResultCompound(
+    name: 'flutter_elements',
+  );
+
+  timer.reset();
+  final graph = HeapSnapshotGraph.fromChunks([bytes.buffer.asByteData()]);
+  print('[+${timer.elapsedMilliseconds} ms] Create HeapSnapshotGraph');
+
+  final analysis = Analysis(graph);
+
+  // Computing reachable objects takes some time.
+  timer.reset();
+  analysis.reachableObjects;
+  print('[+${timer.elapsedMilliseconds} ms] Compute reachable objects');
+  print('');
+  {
+    final measure = analysis.measureObjects(analysis.reachableObjects);
+    allResults.add(
+      BenchmarkResultCompound(name: 'reachableObjects', children: [
+        BenchmarkResultCount(
+          name: 'count',
+          value: measure.count,
+        ),
+        BenchmarkResultBytes(
+          name: 'size',
+          value: measure.size,
+        ),
+      ]),
+    );
+  }
+
+  // It is interesting to see all reachable objects.
+  {
+    timer.reset();
+    print('Reachable objects');
+    final objects = analysis.reachableObjects;
+    analysis.printObjectStats(objects, maxLines: 100);
+    print('');
+  }
+
+  allResults.add(
+    _doUniqueUriStr(analysis),
+  );
+
+  return allResults;
+}
+
+BenchmarkResult _doUniqueUriStr(Analysis analysis) {
+  print('Instances of: _SimpleUri');
+  final uriList = analysis.filterByClass(analysis.reachableObjects,
+      libraryUri: Uri.parse('dart:core'), name: '_SimpleUri');
+  analysis.printObjectStats(uriList);
+  print('');
+
+  final uriStringList = analysis.findReferences(uriList, [':_uri']);
+
+  final uniqueUriStrSet = <String>{};
+  final duplicateUriStrList = <String>[];
+  for (final objectId in uriStringList) {
+    final object = analysis.graph.objects[objectId];
+    final uriStr = object.data as String;
+    if (!uniqueUriStrSet.add(uriStr)) {
+      duplicateUriStrList.add(uriStr);
+    }
+  }
+  print('');
+
+  final uriListMeasure = analysis.measureObjects(uriList);
+  return BenchmarkResultCompound(name: '_SimpleUri', children: [
+    BenchmarkResultCount(
+      name: 'count',
+      value: uriListMeasure.count,
+    ),
+    BenchmarkResultBytes(
+      name: 'size(shallow)',
+      value: uriListMeasure.size,
+    ),
+    BenchmarkResultCount(
+      name: 'duplicateCount',
+      value: duplicateUriStrList.length,
+    ),
+  ]);
+}
+
+/// Loads all libraries available in the analysis contexts, and deserializes
+/// every element in them.
+Future<void> _getAvailableLibraries(
+  AnalysisContextCollectionImpl collection,
+) async {
+  for (final analysisContext in collection.contexts) {
+    final analysisDriver = analysisContext.driver;
+    await analysisDriver.discoverAvailableFiles();
+    final knownFiles = analysisDriver.fsState.knownFiles.toList();
+    for (final file in knownFiles) {
+      // Skip libraries with known invalid types.
+      // if (const {'dart:html', 'dart:ui_web', 'dart:_interceptors'}
+      //     .contains(file.uriStr)) {
+      //   continue;
+      // }
+      final result = await analysisDriver.getLibraryByUri(file.uriStr);
+      if (result is LibraryElementResult) {
+        result.element.accept(_AllElementVisitor());
+      }
+    }
+  }
+}
+
+Uint8List _getHeapSnapshot() {
+  timer.reset();
+  final tmpDir = io.Directory.systemTemp.createTempSync('analyzer_heap');
+  try {
+    final snapshotFile = io.File('${tmpDir.path}/0.heap_snapshot');
+    developer.NativeRuntime.writeHeapSnapshotToFile(snapshotFile.path);
+    print('[+${timer.elapsedMilliseconds} ms] Write heap snapshot');
+
+    timer.reset();
+    final bytes = snapshotFile.readAsBytesSync();
+    print(
+      '[+${timer.elapsedMilliseconds} ms] '
+      'Read heap snapshot, ${bytes.length ~/ (1024 * 1024)} MB',
+    );
+    return bytes;
+  } finally {
+    tmpDir.deleteSync(recursive: true);
+  }
+}
+
+Future<T> _withNewAnalysisContext<T>(
+  Future<T> Function(AnalysisContextCollectionImpl collection) f, {
+  required ByteStore byteStore,
+}) async {
+  final resourceProvider = PhysicalResourceProvider.INSTANCE;
+  final fileContentCache = FileContentCache(resourceProvider);
+  final unlinkedUnitStore = UnlinkedUnitStoreImpl();
+  final collection = AnalysisContextCollectionImpl(
+    byteStore: byteStore,
+    resourceProvider: resourceProvider,
+    fileContentCache: fileContentCache,
+    includedPaths: [includedPath],
+    unlinkedUnitStore: unlinkedUnitStore,
+  );
+  final result = await f(collection);
+  collection.hashCode; // to keep it alive
+  return result;
+}
+
+class _AllElementVisitor extends GeneralizingElementVisitor<void> {
+  @override
+  void visitElement(Element element) {
+    // This triggers lazy reading.
+    element.metadata;
+    super.visitElement(element);
+  }
+}
+
+class _ObjectSetMeasure {
+  final int count;
+  final int size;
+
+  _ObjectSetMeasure({required this.count, required this.size});
+}
+
+extension on Analysis {
+  IntSet filterByClass(
+    IntSet objectIds, {
+    required Uri libraryUri,
+    required String name,
+  }) {
+    final cid = graph.classes.singleWhere((class_) {
+      return class_.libraryUri == libraryUri && class_.name == name;
+    }).classId;
+    return filter(objectIds, (object) => object.classId == cid);
+  }
+
+  _ObjectSetMeasure measureObjects(IntSet objectIds) {
+    final stats = generateObjectStats(objectIds);
+    var totalSize = 0;
+    var totalCount = 0;
+    for (final class_ in stats.classes) {
+      totalCount += stats.counts[class_.classId];
+      totalSize += stats.sizes[class_.classId];
+    }
+    return _ObjectSetMeasure(count: totalCount, size: totalSize);
+  }
+
+  void printObjectStats(IntSet objectIds, {int maxLines = 20}) {
+    final stats = generateObjectStats(objectIds);
+    print(formatHeapStats(stats, maxLines: maxLines));
+    print('');
+  }
+
+  // ignore: unused_element
+  void printRetainers(
+    IntSet objectIds, {
+    int maxEntries = 3,
+  }) {
+    final paths = retainingPathsOf(objectIds, 20);
+    for (int i = 0; i < paths.length; ++i) {
+      if (i >= maxEntries) break;
+      final path = paths[i];
+      print('There are ${path.count} retaining paths of');
+      print(formatRetainingPath(graph, paths[i]));
+      print('');
+    }
+  }
+}

pkg/analyzer/tool/benchmark/heap/result.dart

@@ -0,0 +1,87 @@
+// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import '../../../test/util/tree_string_sink.dart';
+
+String formatSizeInBytes(int value) {
+  final buffer = StringBuffer();
+  buffer.write('$value');
+
+  final kb = value ~/ 1024;
+  if (kb > 0) {
+    buffer.write(' = $kb KB');
+  }
+
+  final mb = kb ~/ 1024;
+  if (mb > 0) {
+    buffer.write(' = $mb MB');
+  }
+
+  final gb = mb / 1024;
+  if (gb >= 1.0) {
+    buffer.write(' = ${gb.toStringAsFixed(2)} GB');
+  }
+
+  return buffer.toString();
+}
+
+void writeBenchmarkResult(TreeStringSink sink, BenchmarkResult result) {
+  switch (result) {
+    case BenchmarkResultBytes():
+      final sizeStr = formatSizeInBytes(result.value);
+      sink.writelnWithIndent('${result.name}: $sizeStr');
+    case BenchmarkResultCompound():
+      sink.writelnWithIndent(result.name);
+      sink.withIndent(() {
+        for (final child in result.children) {
+          writeBenchmarkResult(sink, child);
+        }
+      });
+    case BenchmarkResultCount():
+      sink.writelnWithIndent('${result.name}: ${result.value}');
+  }
+}
+
+sealed class BenchmarkResult {
+  final String name;
+
+  BenchmarkResult({
+    required this.name,
+  });
+}
+
+final class BenchmarkResultBytes extends BenchmarkResult {
+  final int value;
+
+  BenchmarkResultBytes({
+    required super.name,
+    required this.value,
+  });
+}
+
+final class BenchmarkResultCompound extends BenchmarkResult {
+  final List<BenchmarkResult> children = [];
+
+  BenchmarkResultCompound({
+    required super.name,
+    List<BenchmarkResult>? children,
+  }) {
+    if (children != null) {
+      this.children.addAll(children);
+    }
+  }
+
+  void add(BenchmarkResult child) {
+    children.add(child);
+  }
+}
+
+final class BenchmarkResultCount extends BenchmarkResult {
+  final int value;
+
+  BenchmarkResultCount({
+    required super.name,
+    required this.value,
+  });
+}

pkg/heap_snapshot/lib/analysis.dart

@@ -2,6 +2,13 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// ignore_for_file: annotate_overrides
+// ignore_for_file: avoid_init_to_null
+// ignore_for_file: avoid_function_literals_in_foreach_calls
+// ignore_for_file: empty_constructor_bodies
+// ignore_for_file: prefer_is_not_empty
+// ignore_for_file: unnecessary_brace_in_string_interps
+
 import 'dart:typed_data';
 
 import 'package:vm_service/vm_service.dart';

pkg/heap_snapshot/lib/format.dart

@@ -2,6 +2,9 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+// ignore_for_file: prefer_interpolation_to_compose_strings
+// ignore_for_file: unnecessary_string_interpolations
+
 import 'dart:typed_data';
 
 import 'package:vm_service/vm_service.dart';

pkg/heap_snapshot/pubspec.yaml

@@ -0,0 +1,13 @@
+name: heap_snapshot
+# This package is not intended for consumption on pub.dev. DO NOT publish.
+publish_to: none
+
+environment:
+  sdk: '>=2.17.0 <3.0.0'
+
+dependencies:
+  vm_service: any
+
+dependency_overrides:
+  vm_service:
+    path: ../vm_service

pkg/heap_snapshot/test/int_set_benchmark.dart

@@ -2,7 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:heapsnapshot/src/intset.dart';
+import 'package:heap_snapshot/intset.dart';
 
 main() {
   for (int every in [16, 8, 4, 3, 2, 1]) {

pkg/heap_snapshot/test/int_set_test.dart

@@ -4,7 +4,7 @@
 
 import 'dart:math';
 
-import 'package:heapsnapshot/src/intset.dart';
+import 'package:heap_snapshot/intset.dart';
 
 main() {
   checkDump();
@@ -104,7 +104,7 @@ void checkDump() {
 }
 
 void checkIsEmpty(Set<int> set) {
-  if (set.length != 0) throw "Got a non-zero length.";
+  if (set.isNotEmpty) throw "Got a non-zero length.";
   for (int i in set) {
     throw "Iterated and got (at least) '$i', expected empty.";
   }
@@ -163,7 +163,7 @@ void handCodedTests() {
 void randomTests() {
   int seed = Random.secure().nextInt(10000);
   print("Using seed $seed");
-  Random r = new Random(seed);
+  Random r = Random(seed);
   List<int> expect = [];
   const int maxExclusive = 100;
   Set<int> set = SpecializedIntSet(maxExclusive);

runtime/tests/vm/dart/regress_52703_test.dart

@@ -7,11 +7,11 @@ import 'dart:developer';
 import 'dart:io';
 
 import 'package:expect/expect.dart';
+import 'package:heap_snapshot/analysis.dart';
 import 'package:path/path.dart' as path;
 import 'package:vm_service/vm_service.dart';
 
 import 'use_flag_test_helper.dart';
-import '../../../tools/heapsnapshot/lib/src/analysis.dart';
 
 void main() async {
   if (buildDir.contains('Product')) return;

runtime/tools/heapsnapshot/lib/heapsnapshot.dart

@@ -1,5 +0,0 @@
-// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-export 'src/analysis.dart';

runtime/tools/heapsnapshot/lib/src/cli.dart

@@ -7,13 +7,13 @@ import 'dart:async';
 
 import 'package:args/args.dart';
 
+import 'package:heap_snapshot/analysis.dart';
+import 'package:heap_snapshot/format.dart';
 import 'package:mmap/mmap.dart';
 import 'package:vm_service/vm_service.dart';
 
-import 'analysis.dart';
 import 'completion.dart';
 import 'expression.dart';
-import 'format.dart';
 import 'load.dart';
 export 'expression.dart' show Output;
 

runtime/tools/heapsnapshot/lib/src/expression.dart

@@ -4,9 +4,9 @@
 
 import 'dart:math' as math;
 
+import 'package:heap_snapshot/analysis.dart';
 import 'package:vm_service/vm_service.dart';
 
-import 'analysis.dart';
 import 'completion.dart';
 
 abstract class SetExpression {

runtime/tools/heapsnapshot/pubspec.yaml

@@ -11,14 +11,17 @@ environment:
 
 dependencies:
   args: ^2.0.0
-  vm_service: ^10.0.0
+  heap_snapshot: any
   mmap: any
+  vm_service: ^10.0.0
 
 dev_dependencies:
-  test: ^1.21.6
-  path: ^1.8.0
   dart_console: ^1.1.2
+  path: ^1.8.0
+  test: ^1.21.6
 
 dependency_overrides:
+  heap_snapshot:
+    path: ../../../pkg/heap_snapshot
   mmap:
     path: ../../../pkg/mmap

runtime/tools/heapsnapshot/test/completion_test.dart

@@ -4,7 +4,7 @@
 
 import 'dart:io';
 
-import 'package:heapsnapshot/heapsnapshot.dart';
+import 'package:heap_snapshot/analysis.dart';
 import 'package:heapsnapshot/src/cli.dart';
 import 'package:path/path.dart' as path;