diff --git a/runtime/tests/vm/dart/heapsnapshot_cli_test.dart b/runtime/tests/vm/dart/heapsnapshot_cli_test.dart new file mode 100644 index 00000000000..2012834b02c --- /dev/null +++ b/runtime/tests/vm/dart/heapsnapshot_cli_test.dart @@ -0,0 +1,22 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import '../../../tools/heapsnapshot/test/cli_test.dart' as cli_test; +import '../../../tools/heapsnapshot/test/expression_test.dart' as expr_test; + +import 'use_flag_test_helper.dart'; + +main(List args) { + if (!buildDir.contains('Release') || isSimulator) return; + + // The cli_test may launch subprocesses using Platform.script, if it does we + // delegate subprocess logic to it. + if (!args.isEmpty) { + cli_test.main(args); + return; + } + + cli_test.main(args); + expr_test.main(args); +} diff --git a/runtime/tools/heapsnapshot/CHANGELOG.md b/runtime/tools/heapsnapshot/CHANGELOG.md new file mode 100644 index 00000000000..a1168998833 --- /dev/null +++ b/runtime/tools/heapsnapshot/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## 0.1.0 + +- Initial version diff --git a/runtime/tools/heapsnapshot/bin/explore.dart b/runtime/tools/heapsnapshot/bin/explore.dart new file mode 100644 index 00000000000..029060f1f93 --- /dev/null +++ b/runtime/tools/heapsnapshot/bin/explore.dart @@ -0,0 +1,63 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:dart_console/dart_console.dart'; +import 'package:heapsnapshot/src/cli.dart'; + +class ConsoleErrorPrinter extends Output { + final Console console; + + ConsoleErrorPrinter(this.console); + + void printError(String error) { + console.writeErrorLine(error); + } + + void print(String message) { + console.writeLine(message); + } +} + +void main() async { + final console = Console.scrolling(); + + console.write('The '); + console.setForegroundColor(ConsoleColor.brightYellow); + console.write('Dart VM *.heapsnapshot analysis tool'); + console.resetColorAttributes(); + console.writeLine(''); + + console.writeLine('Type `exit` or use Ctrl+D to exit.'); + console.writeLine(''); + + final errors = ConsoleErrorPrinter(console); + final cliState = CliState(errors); + + while (true) { + void writePrompt() { + console.setForegroundColor(ConsoleColor.brightBlue); + console.write('(hsa) '); + console.resetColorAttributes(); + console.setForegroundColor(ConsoleColor.brightGreen); + } + + writePrompt(); + final response = console.readLine(cancelOnEOF: true); + console.resetColorAttributes(); + if (response == null) return; + if (response.isEmpty) continue; + + final args = response + .split(' ') + .map((p) => p.trim()) + .where((p) => !p.isEmpty) + .toList(); + if (args.isEmpty) continue; + if (args.length == 1 && args.single == 'exit') { + return; + } + + await cliCommandRunner.run(cliState, args); + } +} diff --git a/runtime/tools/heapsnapshot/lib/heapsnapshot.dart b/runtime/tools/heapsnapshot/lib/heapsnapshot.dart new file mode 100644 index 00000000000..1af305ed19e --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/heapsnapshot.dart @@ -0,0 +1,5 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +export 'src/analysis.dart'; diff --git a/runtime/tools/heapsnapshot/lib/src/analysis.dart b/runtime/tools/heapsnapshot/lib/src/analysis.dart new file mode 100644 index 00000000000..6e257f92329 --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/src/analysis.dart @@ -0,0 +1,978 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:vm_service/vm_service.dart'; + +import 'format.dart'; + +const int _invalidIdx = 0; +const int _rootObjectIdx = 1; + +class Analysis { + final HeapSnapshotGraph graph; + + late final reachableObjects = transitiveGraph({_rootObjectIdx}); + + late final Uint32List _retainers = _calculateRetainers(); + + late final _oneByteStringCid = _findClassId('_OneByteString'); + late final _twoByteStringCid = _findClassId('_TwoByteString'); + late final _nonGrowableListCid = _findClassId('_List'); + late final _immutableListCid = _findClassId('_ImmutableList'); + late final _weakPropertyCid = _findClassId('_WeakProperty'); + late final _weakReferenceCid = _findClassId('_WeakReferenceImpl'); + late final _patchClassCid = _findClassId('PatchClass'); + late final _finalizerEntryCid = _findClassId('FinalizerEntry'); + + late final _weakPropertyKeyIdx = _findFieldIndex(_weakPropertyCid, 'key_'); + late final _weakPropertyValueIdx = + _findFieldIndex(_weakPropertyCid, 'value_'); + + late final _finalizerEntryDetachIdx = + _findFieldIndex(_finalizerEntryCid, 'detach_'); + late final _finalizerEntryValueIdx = + _findFieldIndex(_finalizerEntryCid, 'value_'); + + late final _Arch _arch = (() { + // We want to figure out the architecture this heapsnapshot was made from + // without it being directly included in the snapshot. + // In order to distinguish 32-bit/64-bit/64-bit-compressed we need + // - an object whose shallowSize will be different for all 3 architectures + // - have an actual object in the heap snapshot + // -> PatchClass seems to satisfy this. + final size = graph.objects + .firstWhere( + (obj) => obj.classId == _patchClassCid && obj.shallowSize != 0) + .shallowSize; + + switch (size) { + case 24: + return _Arch.arch32; + case 32: + return _Arch.arch64c; + case 48: + return _Arch.arch64; + default: + throw 'Unexpected size of patch class: $size.'; + } + })(); + + late final int _headerSize = _arch != _Arch.arch32 ? 8 : 4; + late final int _wordSize = _arch == _Arch.arch64 ? 8 : 4; + + Analysis(this.graph); + + /// The roots from which alive data can be discovered. + Set get roots => {_rootObjectIdx}; + + /// Calculates retaining paths for all objects in [objs]. + /// + /// All retaining paths will have the object itself plus at most [depth] + /// retainers in it. + List retainingPathsOf(Set objs, int depth) { + final paths = {}; + for (var oId in objs) { + final rpath = _retainingPathOf(oId, depth); + final old = paths[rpath]; + paths[rpath] = (old == null) ? 1 : old + 1; + } + paths.forEach((path, count) { + path.count = count; + }); + return paths.keys.toList()..sort((a, b) => paths[b]! - paths[a]!); + } + + /// Returns information about a specific object. + ObjectInformation examine(int oId) { + String stringifyValue(int valueId) { + if (valueId == _invalidIdx) return 'int/double/simd'; + + final object = graph.objects[valueId]; + final cid = object.classId; + if (cid == _oneByteStringCid || cid == _twoByteStringCid) { + return '"${truncateString(object.data as String)}"'; + } + + final valueClass = graph.classes[cid]; + return '${valueClass.name}@${valueId} (${valueClass.libraryUri})'; + } + + final object = graph.objects[oId]; + final cid = object.classId; + final klass = graph.classes[cid]; + final fs = klass.fields.toList()..sort((a, b) => a.index - b.index); + final fieldValues = {}; + if (cid == _oneByteStringCid || cid == _twoByteStringCid) { + fieldValues['data'] = stringifyValue(oId); + } else { + int maxFieldIndex = -1; + for (final field in fs) { + final valueId = object.references[field.index]; + fieldValues[field.name] = stringifyValue(valueId); + if (field.index > maxFieldIndex) { + maxFieldIndex = field.index; + } + } + + if (cid == _immutableListCid || cid == _nonGrowableListCid) { + final refs = object.references; + int len = refs.length - (maxFieldIndex + 1); + if (len < 10) { + for (int i = 0; i < len; ++i) { + fieldValues['[$i]'] = stringifyValue(refs[1 + maxFieldIndex + i]); + } + } else { + for (int i = 0; i < 4; ++i) { + fieldValues['[$i]'] = stringifyValue(refs[1 + maxFieldIndex + i]); + } + fieldValues['[...]'] = ''; + for (int i = len - 4; i < len; ++i) { + fieldValues['[$i]'] = stringifyValue(refs[1 + maxFieldIndex + i]); + } + } + } + } + return ObjectInformation( + klass.name, klass.libraryUri.toString(), fieldValues); + } + + /// Generates statistics about the given set of [objects]. + /// + /// The classes are sored by sum of shallow-size of objects of a class if + /// [sortBySize] is true and by number of objects per-class otherwise. + HeapStats generateObjectStats(Set objects, {bool sortBySize = true}) { + final graphObjects = graph.objects; + final numCids = graph.classes.length; + + final counts = Int32List(numCids); + final sizes = Int32List(numCids); + for (final objectId in objects) { + final obj = graphObjects[objectId]; + final cid = obj.classId; + counts[cid]++; + sizes[cid] += obj.shallowSize; + } + + final classes = graph.classes.where((c) => counts[c.classId] > 0).toList(); + if (sortBySize) { + classes.sort((a, b) { + var diff = sizes[b.classId] - sizes[a.classId]; + if (diff != 0) return diff; + diff = counts[b.classId] - counts[a.classId]; + if (diff != 0) return diff; + return graph.classes[b.classId].name + .compareTo(graph.classes[a.classId].name); + }); + } else { + classes.sort((a, b) { + var diff = counts[b.classId] - counts[a.classId]; + if (diff != 0) return diff; + diff = sizes[b.classId] - sizes[a.classId]; + if (diff != 0) return diff; + return graph.classes[b.classId].name + .compareTo(graph.classes[a.classId].name); + }); + } + + return HeapStats(classes, sizes, counts); + } + + /// Generate statistics about the variable-length data of [objects]. + /// + /// The returned [HeapData]s are sorted by cumulative size if + /// [sortBySize] is true and by number of objects otherwise. + HeapDataStats generateDataStats(Set objects, {bool sortBySize = true}) { + final graphObjects = graph.objects; + final klasses = graph.classes; + final counts = {}; + for (final objectId in objects) { + final obj = graphObjects[objectId]; + final klass = klasses[obj.classId].name; + // Should use length here instead! + final len = variableLengthOf(obj); + if (len == -1) continue; + final data = HeapData(klass, obj.data, obj.shallowSize, len); + counts[data] = (counts[data] ?? 0) + 1; + } + counts.forEach((HeapData data, int count) { + data.count = count; + }); + + final datas = counts.keys.toList(); + if (sortBySize) { + datas.sort((a, b) => b.totalSize - a.totalSize); + } else { + datas.sort((a, b) => b.count - a.count); + } + + return HeapDataStats(datas); + } + + /// Calculates the set of objects transitively reachable by [roots]. + Set transitiveGraph(Set roots, [TraverseFilter? tfilter = null]) { + final reachable = {}; + final worklist = []; + + final objects = graph.objects; + + reachable.addAll(roots); + worklist.addAll(roots); + + final weakProperties = {}; + + while (worklist.isNotEmpty) { + while (worklist.isNotEmpty) { + final objectIdToExpand = worklist.removeLast(); + final objectToExpand = objects[objectIdToExpand]; + final cid = objectToExpand.classId; + + // Weak references don't keep their value alive. + if (cid == _weakReferenceCid) continue; + + // Weak properties keep their value alive if the key is alive. + if (cid == _weakPropertyCid) { + if (tfilter == null || + tfilter._shouldTraverseEdge( + _weakPropertyCid, _weakPropertyValueIdx)) { + weakProperties.add(objectIdToExpand); + } + continue; + } + + // Normal object (or FinalizerEntry). + final references = objectToExpand.references; + final bool isFinalizerEntry = cid == _finalizerEntryCid; + for (int i = 0; i < references.length; ++i) { + // [FinalizerEntry] objects don't keep their "detach" and "value" + // fields alive. + if (isFinalizerEntry && + (i == _finalizerEntryDetachIdx || i == _finalizerEntryValueIdx)) { + continue; + } + + final successor = references[i]; + if (!reachable.contains(successor)) { + if (tfilter == null || + (tfilter._shouldTraverseEdge(objectToExpand.classId, i) && + tfilter._shouldIncludeObject(objects[successor].classId))) { + reachable.add(successor); + worklist.add(successor); + } + } + } + } + + // Enqueue values of weak properties if their key is alive. + weakProperties.removeWhere((int weakProperty) { + final wpReferences = objects[weakProperty].references; + final keyId = wpReferences[_weakPropertyKeyIdx]; + final valueId = wpReferences[_weakPropertyValueIdx]; + if (reachable.contains(keyId)) { + if (!reachable.contains(valueId)) { + if (tfilter == null || + tfilter._shouldIncludeObject(objects[valueId].classId)) { + reachable.add(valueId); + worklist.add(valueId); + } + } + return true; + } + return false; + }); + } + return reachable; + } + + /// Calculates the set of objects that transitively can reach [oids]. + Set reverseTransitiveGraph(Set oids, + [TraverseFilter? tfilter = null]) { + final reachable = {}; + final worklist = []; + + final objects = graph.objects; + + reachable.addAll(oids); + worklist.addAll(oids); + + while (worklist.isNotEmpty) { + final objectIdToExpand = worklist.removeLast(); + final objectToExpand = objects[objectIdToExpand]; + final referrers = objectToExpand.referrers; + for (int i = 0; i < referrers.length; ++i) { + final predecessorId = referrers[i]; + // This is a dead object in heap that refers to a live object. + if (!reachableObjects.contains(predecessorId)) continue; + if (!reachable.contains(predecessorId)) { + final predecessor = objects[predecessorId]; + final cid = predecessor.classId; + + // A WeakReference does not keep its object alive. + if (cid == _weakReferenceCid) continue; + + // A WeakProperty does not keep its key alive, but may keep it's value + // alive. + if (cid == _weakPropertyCid) { + final refs = predecessor.references; + bool hasRealRef = false; + for (int i = 0; i < refs.length; ++i) { + if (i == _weakPropertyKeyIdx) continue; + if (refs[i] == objectIdToExpand) hasRealRef = true; + } + if (!hasRealRef) continue; + } + + // A FinalizerEntry] does not keep its {detach_,value_} fields alive. + if (cid == _finalizerEntryCid) { + final refs = predecessor.references; + bool hasRealRef = false; + for (int i = 0; i < refs.length; ++i) { + if (i == _finalizerEntryDetachIdx) continue; + if (i == _finalizerEntryValueIdx) continue; + if (refs[i] == objectIdToExpand) hasRealRef = true; + } + if (!hasRealRef) continue; + } + + bool passedFilter = true; + if (tfilter != null) { + final index = predecessor.references.indexOf(objectIdToExpand); + passedFilter = + (tfilter._shouldTraverseEdge(predecessor.classId, index) && + tfilter._shouldIncludeObject(predecessor.classId)); + } + if (passedFilter) { + reachable.add(predecessorId); + worklist.add(predecessorId); + } + } + } + } + return reachable; + } + + // Only keep those in [toFilter] that have references from [from]. + Set filterObjectsReferencedBy(Set toFilter, Set from) { + final result = {}; + final objects = graph.objects; + + for (final fromId in from) { + final from = objects[fromId]; + for (final refId in from.references) { + if (toFilter.contains(refId)) { + result.add(refId); + break; + } + } + } + + return result; + } + + /// Returns set of cids that are matching the provided [patterns]. + Set findClassIdsMatching(Iterable patterns) { + final regexPatterns = patterns.map((p) => RegExp(p)).toList(); + + final classes = graph.classes; + final cids = {}; + for (final klass in classes) { + if (regexPatterns.any((pattern) => + pattern.hasMatch(klass.name) || + pattern.hasMatch(klass.libraryUri.toString()))) { + cids.add(klass.classId); + } + } + return cids; + } + + /// Create filters that can be used in traversing object graphs. + TraverseFilter? parseTraverseFilter(List patterns) { + if (patterns.isEmpty) return null; + + final aset = {}; + final naset = {}; + + int bits = 0; + + final fmap = >{}; + final nfmap = >{}; + for (String pattern in patterns) { + final bool isNegated = pattern.startsWith('^'); + if (isNegated) { + pattern = pattern.substring(1); + } + + // Edge filter. + final int sep = pattern.indexOf(':'); + if (sep != -1 && sep != (pattern.length - 1)) { + final klassPattern = pattern.substring(0, sep); + + final fieldNamePattern = pattern.substring(sep + 1); + final cids = findClassIdsMatching([klassPattern]); + + final fieldNameRegexp = RegExp(fieldNamePattern); + for (final cid in cids) { + final klass = graph.classes[cid]; + for (final field in klass.fields) { + if (fieldNameRegexp.hasMatch(field.name)) { + (isNegated ? nfmap : fmap) + .putIfAbsent(cid, _buildSet) + .add(field.index); + } + } + } + + if (!isNegated) { + bits |= TraverseFilter._hasPositiveEdgePatternBit; + } + + continue; + } + + // Class filter. + final cids = findClassIdsMatching([pattern]); + (isNegated ? naset : aset).addAll(cids); + + if (!isNegated) { + bits |= TraverseFilter._hasPositiveClassPatternBit; + } + } + return TraverseFilter._(patterns, bits, aset, naset, fmap, nfmap); + } + + /// Returns set of objects from [objectIds] whose class id is in [cids]. + Set filterByClassId(Set objectIds, Set cids) { + return filter(objectIds, (object) => cids.contains(object.classId)); + } + + /// Returns set of objects from [objectIds] whose class id is in [cids]. + Set filterByClassPatterns(Set objectIds, List patterns) { + final tfilter = parseTraverseFilter(patterns); + if (tfilter == null) return objectIds; + return filter(objectIds, tfilter._shouldFilterObject); + } + + /// Returns set of objects from [objectIds] whose class id is in [cids]. + Set filter( + Set objectIds, bool Function(HeapSnapshotObject) filter) { + final result = {}; + final objects = graph.objects; + objectIds.forEach((int objId) { + if (filter(objects[objId])) { + result.add(objId); + } + }); + return result; + } + + /// Returns users of [objs]. + Set findUsers(Set objs, List patterns) { + final tfilter = parseTraverseFilter(patterns); + + final objects = graph.objects; + final result = {}; + for (final objId in objs) { + final object = objects[objId]; + final referrers = object.referrers; + for (int i = 0; i < referrers.length; ++i) { + final userId = referrers[i]; + // This is a dead object in heap that refers to a live object. + if (!reachableObjects.contains(userId)) continue; + bool passedFilter = true; + if (tfilter != null) { + final user = objects[userId]; + final idx = user.references.indexOf(objId); + passedFilter = tfilter._shouldTraverseEdge(user.classId, idx) && + tfilter._shouldIncludeObject(user.classId); + } + if (passedFilter) { + result.add(userId); + } + } + } + return result; + } + + /// Returns references of [objs]. + Set findReferences(Set objs, List patterns) { + final tfilter = parseTraverseFilter(patterns); + + final objects = graph.objects; + final result = {}; + for (final objId in objs) { + final object = objects[objId]; + final references = object.references; + for (int i = 0; i < references.length; ++i) { + final refId = references[i]; + bool passedFilter = true; + if (tfilter != null) { + final other = objects[refId]; + passedFilter = tfilter._shouldTraverseEdge(object.classId, i) && + tfilter._shouldIncludeObject(other.classId); + } + if (passedFilter) { + result.add(refId); + } + } + } + return result; + } + + /// Returns the size of the variable part of [object] + /// + /// For strings this is the length of the string (or approximation thereof). + /// For typed data this is the number of elements. + /// For fixed-length arrays this is the length of the array. + int variableLengthOf(HeapSnapshotObject object) { + final cid = object.classId; + + final isList = cid == _nonGrowableListCid || cid == _immutableListCid; + if (isList) { + // Return the length of the non-growable array. + final numFields = graph.classes[cid].fields.length; + return object.references.length - numFields; + } + + final isString = cid == _oneByteStringCid || cid == _twoByteStringCid; + if (isString) { + // Return the length of the string. + // + // - For lengths <128 the length of string is precise + // - For larger strings, the data is truncated, so we use the payload + // size. + // - TODO: The *heapsnapshot format contains actual length but it gets + // lost after reading. Can we preserve it somewhere on + // `HeapSnapshotGraph`? + // + // The approximation is based on knowning the header size of a string: + // - String has: header, length (hash - on 32-bit platforms) + payload + final fixedSize = + _headerSize + _wordSize * (_arch == _Arch.arch32 ? 2 : 1); + final len = + object.shallowSize == 0 ? 0 : (object.shallowSize - fixedSize); + if (len < 128) return (object.data as String).length; + return len; // Over-approximates to 2 * wordsize. + } + + final data = object.data; + if (data is HeapSnapshotObjectLengthData) { + // Most likely typed data object, return length in elements. + return data.length; + } + + final fixedSize = _headerSize + _wordSize * object.references.length; + final dataSize = object.shallowSize - fixedSize; + if (dataSize > _wordSize) { + final klass = graph.classes[cid]; + // User-visible, but VM-recognized objects with variable size. + if (!['_RegExp', '_SuspendState'].contains(klass.name)) { + // Non-user-visible, VM-recognized objects (empty library uri). + final uri = klass.libraryUri.toString().trim(); + if (uri != '') { + throw 'Object has fixed size: $fixedSize and total ' + 'size: ${object.shallowSize} but is not known to ' + 'be variable-length (class: ${graph.classes[cid].name})'; + } + } + } + + return -1; + } + + int _findClassId(String className) { + return graph.classes + .singleWhere((klass) => + klass.name == className && + (klass.libraryUri.scheme == 'dart' || + klass.libraryUri.toString() == '')) + .classId; + } + + int _findFieldIndex(int cid, String fieldName) { + return graph.classes[cid].fields + .singleWhere((f) => f.name == fieldName) + .index; + } + + DedupedUint32List _retainingPathOf(int oId, int depth) { + final objects = graph.objects; + final classes = graph.classes; + + @pragma('vm:prefer-inline') + int getFieldIndex(int oId, int childId) { + final object = objects[oId]; + final fields = classes[object.classId].fields; + final idx = object.references.indexOf(childId); + if (idx == -1) throw 'should not happen'; + + int fieldIndex = fields.any((f) => f.index == idx) + ? idx + : DedupedUint32List.noFieldIndex; + return fieldIndex; + } + + @pragma('vm:prefer-inline') + int retainingPathLength(int id) { + int length = 1; + int id = oId; + while (id != _rootObjectIdx && length <= depth) { + id = _retainers[id]; + length++; + } + return length; + } + + @pragma('vm:prefer-inline') + bool hasMoreThanOneAlive(Set reachableObjects, Uint32List list) { + int count = 0; + for (int i = 0; i < list.length; ++i) { + if (reachableObjects.contains(list[i])) { + count++; + if (count >= 2) return true; + } + } + return false; + } + + int lastId = oId; + var lastObject = objects[lastId]; + + final path = Uint32List(2 * retainingPathLength(oId) - 1); + path[0] = lastObject.classId; + for (int i = 1; i < path.length; i += 2) { + assert(lastId != _rootObjectIdx && ((i - 1) ~/ 2) < depth); + final users = lastObject.referrers; + final int userId = _retainers[lastId]; + + final user = objects[userId]; + int fieldIndex = getFieldIndex(userId, lastId); + final lastWasUniqueRef = !hasMoreThanOneAlive(reachableObjects, users); + + path[i] = (lastWasUniqueRef ? 1 : 0) << 0 | fieldIndex << 1; + path[i + 1] = user.classId; + + lastId = userId; + lastObject = user; + } + return DedupedUint32List(path); + } + + Uint32List _calculateRetainers() { + final retainers = Uint32List(graph.objects.length); + + var worklist = {_rootObjectIdx}; + while (!worklist.isEmpty) { + final next = {}; + + for (final objId in worklist) { + final object = graph.objects[objId]; + final cid = object.classId; + + // Weak references don't keep their value alive. + if (cid == _weakReferenceCid) continue; + + // Weak properties keep their value alive if the key is alive. + if (cid == _weakPropertyCid) { + final valueId = object.references[_weakPropertyValueIdx]; + if (reachableObjects.contains(valueId)) { + if (retainers[valueId] == 0) { + retainers[valueId] = objId; + next.add(valueId); + } + } + continue; + } + + // Normal object (or FinalizerEntry). + final references = object.references; + final bool isFinalizerEntry = cid == _finalizerEntryCid; + for (int i = 0; i < references.length; ++i) { + // [FinalizerEntry] objects don't keep their "detach" and "value" + // fields alive. + if (isFinalizerEntry && + (i == _finalizerEntryDetachIdx || i == _finalizerEntryValueIdx)) { + continue; + } + + final refId = references[i]; + if (retainers[refId] == 0) { + retainers[refId] = objId; + next.add(refId); + } + } + } + worklist = next; + } + return retainers; + } +} + +class TraverseFilter { + static const int _hasPositiveClassPatternBit = (1 << 0); + static const int _hasPositiveEdgePatternBit = (1 << 1); + + final List _patterns; + + final int _bits; + + final Set? _allowed; + final Set? _disallowed; + + final Map>? _followMap; + final Map>? _notFollowMap; + + const TraverseFilter._(this._patterns, this._bits, this._allowed, + this._disallowed, this._followMap, this._notFollowMap); + + bool get _hasPositiveClassPattern => + (_bits & _hasPositiveClassPatternBit) != 0; + bool get _hasPositiveEdgePattern => (_bits & _hasPositiveEdgePatternBit) != 0; + + String asString(HeapSnapshotGraph graph) { + final sb = StringBuffer(); + sb.writeln( + 'The traverse filter expression "${_patterns.join(' ')}" matches:\n'); + + final ca = _allowed ?? const {}; + final cna = _disallowed ?? const {}; + + final klasses = graph.classes.toList() + ..sort((a, b) => a.name.compareTo(b.name)); + + for (final klass in klasses) { + final cid = klass.classId; + + final posEdge = []; + final negEdge = []; + + final f = _followMap?[cid] ?? const {}; + final nf = _notFollowMap?[cid] ?? const {}; + for (final field in klass.fields) { + final fieldIndex = field.index; + if (f.contains(fieldIndex)) { + posEdge.add(field.name); + } + if (nf.contains(fieldIndex)) { + negEdge.add(field.name); + } + } + + bool printedClass = false; + final name = klass.name; + if (ca.contains(cid)) { + sb.writeln('[+] $name'); + printedClass = true; + } + if (cna.contains(cid)) { + sb.writeln('[-] $name'); + printedClass = true; + } + if (posEdge.isNotEmpty || negEdge.isNotEmpty) { + if (!printedClass) { + sb.writeln('[ ] $name'); + printedClass = true; + } + for (final field in posEdge) { + sb.writeln('[+] .$field'); + } + for (final field in negEdge) { + sb.writeln('[-] .$field'); + } + } + } + return sb.toString().trim(); + } + + // Should include the edge when building transitive graphs. + bool _shouldTraverseEdge(int cid, int fieldIndex) { + final nf = _notFollowMap?[cid]; + if (nf != null && nf.contains(fieldIndex)) return false; + + final f = _followMap?[cid]; + if (f != null && f.contains(fieldIndex)) return true; + + // If there's an allow list we only allow allowed ones, otherwise we allow + // all. + return !_hasPositiveEdgePattern; + } + + // Should include the object when building transitive graphs. + bool _shouldIncludeObject(int cid) { + if (_disallowed?.contains(cid) == true) return false; + if (_allowed?.contains(cid) == true) return true; + + // If there's an allow list we only allow allowed ones, otherwise we allow + // all. + return !_hasPositiveClassPattern; + } + + // Should include the object when filtering a set of objects. + bool _shouldFilterObject(HeapSnapshotObject object) { + final cid = object.classId; + final numReferences = object.references.length; + return __shouldFilterObject(cid, numReferences); + } + + bool __shouldFilterObject(int cid, int numReferences) { + if (!_shouldIncludeObject(cid)) return false; + + // Check if the object has an explicitly disallowed field. + final nf = _notFollowMap?[cid]; + if (nf != null) { + for (int fieldIndex = 0; fieldIndex < numReferences; ++fieldIndex) { + if (nf.contains(fieldIndex)) return false; + } + } + + // Check if the object has an explicitly allowed field. + final f = _followMap?[cid]; + if (f != null) { + for (int fieldIndex = 0; fieldIndex < numReferences; ++fieldIndex) { + if (f.contains(fieldIndex)) return true; + } + } + + // If there's an allow list we only allow allowed ones, otherwise we allow + // all. + return !_hasPositiveEdgePattern; + } +} + +/// Stringified representation of a heap object. +class ObjectInformation { + final String className; + final String libraryUri; + final Map fieldValues; + + ObjectInformation(this.className, this.libraryUri, this.fieldValues); +} + +/// Heap usage statistics calculated for a set of heap objects. +class HeapStats { + final List classes; + final Int32List sizes; + final Int32List counts; + + HeapStats(this.classes, this.sizes, this.counts); + + int get totalSize => sizes.fold(0, (int a, int b) => a + b); + int get totalCount => counts.fold(0, (int a, int b) => a + b); +} + +/// Heap object data statistics calculated for a set of heap objects. +class HeapDataStats { + final List datas; + + HeapDataStats(this.datas); + + int get totalSizeUniqueDatas => + datas.fold(0, (int sum, HeapData d) => sum + d.size); + int get totalSize => + datas.fold(0, (int sum, HeapData d) => sum + d.totalSize); + int get totalCount => datas.fold(0, (int sum, HeapData d) => sum + d.count); +} + +/// Representing the data of one heap object. +/// +/// Since the data can be truncated, it has an extra size that allows to +/// distinguish datas with same truncated value with high probability. +class HeapData { + final String klass; + final dynamic value; + final int size; + final int len; + + late final int count; + + HeapData(this.klass, this.value, this.size, this.len); + + int? _hashCode; + int get hashCode { + if (_hashCode != null) return _hashCode!; + + var valueToHash = value; + if (valueToHash is! String && + valueToHash is! bool && + valueToHash is! double) { + if (valueToHash is HeapSnapshotObjectLengthData) { + valueToHash = valueToHash.length; + } else if (valueToHash is HeapSnapshotObjectNoData) { + valueToHash = 0; + } else if (valueToHash is HeapSnapshotObjectNullData) { + valueToHash = 0; + } else { + throw '${valueToHash.runtimeType}'; + } + } + + return _hashCode = Object.hash(klass, valueToHash, size, len); + } + + bool operator ==(other) { + if (identical(this, other)) return true; + if (other is! HeapData) return false; + if (size != other.size) return false; + if (len != other.len) return false; + if (klass != other.klass) return false; + + final ovalue = other.value; + if (value is String || value is bool || value is double) { + return value == ovalue; + } + // We don't have the typed data content, so we don't know whether they are + // equal / dedupable. + return false; + } + + String get valueAsString { + var d = value; + if (d is String) { + final newLine = d.indexOf('\n'); + if (newLine >= 0) { + d = d.substring(0, newLine); + } + if (d.length > 80) { + d = d.substring(0, 80); + } + return d; + } + return 'len:$len'; + } + + int get totalSize => size * count; +} + +/// Used to represent retaining paths. +/// +/// For retaining paths: `[cid0, fieldIdx1 << 1 | isUniqueOwner, cid1, ...]` +class DedupedUint32List { + static const int noFieldIndex = (1 << 29); + + final Uint32List path; + late final int count; + + DedupedUint32List(this.path); + + int? _hashCode; + int get hashCode => _hashCode ??= Object.hashAll(path); + + bool operator ==(other) { + if (identical(this, other)) return true; + if (other is! DedupedUint32List) return false; + if (path.length != other.path.length) return false; + for (int i = 0; i < path.length; ++i) { + if (path[i] != other.path[i]) return false; + } + return true; + } +} + +enum _Arch { + arch32, + arch64, + arch64c, +} + +Set _buildSet() => {}; diff --git a/runtime/tools/heapsnapshot/lib/src/cli.dart b/runtime/tools/heapsnapshot/lib/src/cli.dart new file mode 100644 index 00000000000..e723f36b5f9 --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/src/cli.dart @@ -0,0 +1,418 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; +import 'dart:async'; + +import 'package:args/args.dart'; + +import 'package:vm_service/vm_service.dart'; + +import 'analysis.dart'; +import 'expression.dart'; +import 'format.dart'; +import 'load.dart'; +export 'expression.dart' show Output; + +abstract class Command { + String get name; + String get description; + String get usage; + List get nameAliases => const []; + + final ArgParser argParser = ArgParser(); + + Future execute(CliState state, List allArgs) async { + try { + int startOfRest = 0; + while (startOfRest < allArgs.length && + allArgs[startOfRest].startsWith('-')) { + startOfRest++; + } + + final options = argParser.parse(allArgs.take(startOfRest).toList()); + final args = allArgs.skip(startOfRest).toList(); + await executeInternal(state, options, args); + } catch (e, s) { + state.output.print('An error occured: $e\n$s'); + printUsage(state); + } + } + + Future executeInternal(CliState state, ArgResults options, List args); + + void printUsage(CliState state) { + if (nameAliases.isEmpty) { + state.output.print('Usage for $name:'); + } else { + state.output + .print('Usage for $name (aliases: ${nameAliases.join(' ')}):'); + } + state.output.print(' $usage'); + } +} + +abstract class SnapshotCommand extends Command { + SnapshotCommand(); + + Future executeInternal( + CliState state, ArgResults options, List args) async { + if (!state.isInitialized) { + state.output.print('No `*.heapsnapshot` loaded. See `help load`.'); + return; + } + await executeSnapshotCommand(state, options, args); + } + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args); +} + +class LoadCommand extends Command { + final name = 'load'; + final description = 'Loads a *.heapsnapshot produced by the Dart VM.'; + final usage = 'load '; + + LoadCommand(); + + Future executeInternal( + CliState state, ArgResults options, List args) async { + HeapSnapshotGraph.getSnapshot; + + if (args.length != 1) { + printUsage(state); + return; + } + final url = args.single; + if (url.startsWith('http') || url.startsWith('ws')) { + try { + final chunks = await loadFromUri(Uri.parse(url)); + state.initialize(Analysis(HeapSnapshotGraph.fromChunks(chunks))); + state.output.print('Loaded heapsnapshot from "$url".'); + } catch (e) { + state.output.print('Could not load heapsnapshot from "$url".'); + } + return; + } + + final filename = url.startsWith('~/') + ? (Platform.environment['HOME']! + url.substring(1)) + : url; + if (!File(filename).existsSync()) { + state.output.print('File "$filename" doesn\'t exist.'); + return; + } + try { + final bytes = File(filename).readAsBytesSync(); + state.initialize( + Analysis(HeapSnapshotGraph.fromChunks([bytes.buffer.asByteData()]))); + state.output.print('Loaded heapsnapshot from "$filename".'); + } catch (e) { + state.output.print('Could not load heapsnapshot from "$filename".'); + return; + } + } +} + +class StatsCommand extends SnapshotCommand { + final name = 'stats'; + final description = 'Calculates statistics about a set of objects.'; + final usage = 'stats [-n/--max=NUM] [-c/--sort-by-count] '; + final nameAliases = ['stat']; + + StatsCommand() { + argParser.addFlag('sort-by-count', + abbr: 'c', + help: 'Sorts by count (instead of size).', + defaultsTo: false); + argParser.addOption('max', + abbr: 'n', + help: 'Limits the number of lines to be printed.', + defaultsTo: '20'); + } + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final oids = parseAndEvaluate( + state.namedSets, state.analysis, args.join(' '), state.output); + if (oids == null) return; + + final sortByCount = options['sort-by-count'] as bool; + final lines = int.parse(options['max']!); + + final stats = + state.analysis.generateObjectStats(oids, sortBySize: !sortByCount); + state.output.print(formatHeapStats(stats, maxLines: lines)); + } +} + +class DataStatsCommand extends SnapshotCommand { + final name = 'dstats'; + final description = + 'Calculates statistics about the data portion of objects.'; + final usage = 'dstats [-n/--max=NUM] [-c/--sort-by-count] '; + final nameAliases = ['dstat']; + + DataStatsCommand() { + argParser.addFlag('sort-by-count', + abbr: 'c', help: 'Sort by count', defaultsTo: false); + argParser.addOption('max', + abbr: 'n', + help: 'Limits the number of max to be printed.', + defaultsTo: '20'); + } + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final oids = parseAndEvaluate( + state.namedSets, state.analysis, args.join(' '), state.output); + if (oids == null) return; + + final sortByCount = options['sort-by-count'] as bool; + final lines = int.parse(options['max']!); + + final stats = + state.analysis.generateDataStats(oids, sortBySize: !sortByCount); + state.output.print(formatDataStats(stats, maxLines: lines)); + } +} + +class InfoCommand extends SnapshotCommand { + final name = 'info'; + final description = 'Prints the known named sets.'; + final usage = 'info'; + + InfoCommand(); + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + if (args.length != 0) { + printUsage(state); + return; + } + + state.output.print('Known named sets:'); + final table = Table(); + state.namedSets.forEach((String name, Set oids) { + table.addRow([name, '{#${oids.length}}']); + }); + state.output.print(indent(' ', table.asString)); + } +} + +class ClearCommand extends SnapshotCommand { + final name = 'clear'; + final description = 'Clears a specific named set (or all).'; + final usage = 'clear *'; + + ClearCommand(); + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + if (args.isEmpty) { + state.namedSets.clearWhere((key) => key != 'roots'); + return; + } + + for (final arg in args) { + state.namedSets.clear(arg); + return; + } + } +} + +class RetainingPathCommand extends SnapshotCommand { + final name = 'retainers'; + final description = 'Prints information about retaining paths.'; + final usage = 'retainers [-d/--depth=] [-n/--max=NUM] '; + final nameAliases = ['retain']; + + RetainingPathCommand() { + argParser.addOption('depth', + abbr: 'd', help: 'Maximum depth of retaining paths.', defaultsTo: '10'); + argParser.addOption('max', + abbr: 'n', + help: 'Limits the number of entries printed.', + defaultsTo: '3'); + } + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final oids = parseAndEvaluate( + state.namedSets, state.analysis, args.join(' '), state.output); + if (oids == null) return; + + final depth = int.parse(options['depth']!); + final maxEntries = int.parse(options['max']!); + + final paths = state.analysis.retainingPathsOf(oids, depth); + for (int i = 0; i < paths.length; ++i) { + if (maxEntries != -1 && i >= maxEntries) break; + final path = paths[i]; + state.output.print('There are ${path.count} retaining paths of'); + state.output.print(formatRetainingPath(state.analysis.graph, paths[i])); + state.output.print(''); + } + } +} + +class ExamineCommand extends SnapshotCommand { + final name = 'examine'; + final description = 'Examins a set of objects.'; + final usage = 'examine [-n/--max=NUM] ?'; + final nameAliases = ['x']; + + ExamineCommand() { + argParser.addOption('max', + abbr: 'n', + help: 'Limits the number of entries to be examined..', + defaultsTo: '5'); + } + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final limit = int.parse(options['max']!); + + final oids = parseAndEvaluate( + state.namedSets, state.analysis, args.join(' '), state.output); + if (oids == null) return; + if (oids.isEmpty) return; + + final it = oids.iterator; + int i = 0; + while (it.moveNext()) { + final oid = it.current; + final info = state.analysis.examine(oid); + state.output.print('${info.className}@$oid (${info.libraryUri}) {'); + final table = Table(); + info.fieldValues.forEach((name, value) { + table.addRow([name, value]); + }); + state.output.print(indent(' ', table.asString)); + state.output.print('}'); + if (++i >= limit) break; + } + } +} + +class EvaluateCommand extends SnapshotCommand { + final name = 'eval'; + final description = 'Evaluates a set expression.'; + final usage = 'eval \n\n$dslDescription'; + + EvaluateCommand(); + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final sexpr = parseExpression( + args.join(' '), state.output, state.namedSets.names.toSet()); + if (sexpr == null) return null; + + final oids = sexpr.evaluate(state.namedSets, state.analysis, state.output); + if (oids == null) return null; + + late final String name; + if (sexpr is SetNameExpression) { + name = sexpr.name; + } else { + name = state.namedSets.nameSet(oids); + } + state.output.print(' $name {#${oids.length}}'); + } +} + +class DescFilterCommand extends SnapshotCommand { + final name = 'describe-filter'; + final description = 'Describes what a filter expression will match.'; + final usage = 'describe-filter $dslFilter'; + final nameAliases = ['desc-filter', 'desc']; + + DescFilterCommand(); + + Future executeSnapshotCommand( + CliState state, ArgResults options, List args) async { + final tfilter = state.analysis.parseTraverseFilter(args); + if (tfilter == null) return null; + + state.output.print(tfilter.asString(state.analysis.graph)); + } +} + +class CommandRunner { + final Command defaultCommand; + final Map name2command = {}; + + CommandRunner(List commands, this.defaultCommand) { + for (final command in commands) { + name2command[command.name] = command; + for (final alias in command.nameAliases) { + name2command[alias] = command; + } + } + } + + Future run(CliState state, List args) async { + if (args.isEmpty) return; + + final commandName = args.first; + final command = name2command[commandName]; + if (command != null) { + await command.execute(state, args.skip(1).toList()); + return; + } + if (const ['help', 'h'].contains(commandName)) { + if (args.length > 1) { + final helpCommandName = args[1]; + final helpCommand = name2command[helpCommandName]; + if (helpCommand != null) { + helpCommand.printUsage(state); + return; + } + } + final table = Table(); + name2command.forEach((name, command) { + if (name == command.name) { + table.addRow([command.name, command.description]); + } + }); + state.output.print('Available commands:'); + state.output.print(indent(' ', table.asString)); + return; + } + + await defaultCommand.execute(state, args); + } +} + +class CliState { + NamedSets? _namedSets; + Analysis? _analysis; + Output output; + + CliState(this.output); + + void initialize(Analysis analysis) { + _analysis = analysis; + + _namedSets = NamedSets(); + _namedSets!.nameSet(analysis.roots, 'roots'); + } + + bool get isInitialized => _analysis != null; + + Analysis get analysis => _analysis!; + NamedSets get namedSets => _namedSets!; +} + +final cliCommandRunner = CommandRunner([ + LoadCommand(), + StatsCommand(), + DataStatsCommand(), + InfoCommand(), + ClearCommand(), + RetainingPathCommand(), + EvaluateCommand(), + ExamineCommand(), + DescFilterCommand(), +], EvaluateCommand()); diff --git a/runtime/tools/heapsnapshot/lib/src/expression.dart b/runtime/tools/heapsnapshot/lib/src/expression.dart new file mode 100644 index 00000000000..ec8c060fa4d --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/src/expression.dart @@ -0,0 +1,668 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:math' as math; + +import 'package:vm_service/vm_service.dart'; + +import 'analysis.dart'; + +abstract class SetExpression { + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output); +} + +class FilterExpression extends SetExpression { + final SetExpression expr; + final List patterns; + + FilterExpression(this.expr, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = expr.evaluate(namedSets, analysis, output); + if (oids == null) return null; + + final patterns = this + .patterns + .map((String pattern) { + if (pattern == 'String') { + return ['_OneByteString', '_TwoByteString']; + } else if (pattern == 'List') { + return ['_List', '_GrowableList', '_ImmutableList']; + } else if (pattern == 'Map') { + return [ + '_HashMap', + '_CompactLinkedHashSet', + '_CompactImmutableLinkedHashSet', + '_InternalLinkedHashMap', + '_InternalImmutableLinkedHashMap' + ]; + } + return [pattern]; + }) + .expand((l) => l) + .toList(); + + return analysis.filterByClassPatterns(oids, patterns); + } +} + +class DFilterExpression extends SetExpression { + final SetExpression expr; + final List patterns; + + DFilterExpression(this.expr, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = expr.evaluate(namedSets, analysis, output); + if (oids == null) return null; + final predicates = patterns.map((String pattern) { + final l = pattern.startsWith('<'); + final le = pattern.startsWith('<='); + final e = pattern.startsWith('=='); + final ge = pattern.startsWith('>='); + final g = pattern.startsWith('>'); + + if (l || le || e || ge || g) { + final value = pattern.substring((le || e || ge) ? 2 : 1); + int limit = int.parse(value); + + if (l) + return (o) { + final len = analysis.variableLengthOf(o); + return len != -1 && len < limit; + }; + if (le) + return (o) { + final len = analysis.variableLengthOf(o); + return len != -1 && len <= limit; + }; + if (e) + return (o) { + final len = analysis.variableLengthOf(o); + return len != -1 && len == limit; + }; + if (ge) + return (o) { + final len = analysis.variableLengthOf(o); + return len != -1 && len >= limit; + }; + if (ge) + return (o) { + final len = analysis.variableLengthOf(o); + return len != -1 && len > limit; + }; + throw 'unreachable'; + } + + if (pattern.startsWith('^')) { + pattern = pattern.substring(1); + final regexp = RegExp(pattern); + return (HeapSnapshotObject object) { + final data = object.data; + if (data is String) { + return !regexp.hasMatch(data); + } + return false; + }; + } + + final regexp = RegExp(pattern); + return (HeapSnapshotObject object) { + final data = object.data; + if (data is String) { + return regexp.hasMatch(data); + } + return false; + }; + }).toList(); + + return analysis.filter( + oids, (object) => !predicates.any((p) => !p(object))); + } +} + +class MinusExpression extends SetExpression { + final SetExpression expr; + final List operands; + + MinusExpression(this.expr, this.operands); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final result = expr.evaluate(namedSets, analysis, output)?.toSet(); + if (result == null) return null; + + for (int i = 0; i < operands.length; ++i) { + final oids = operands[i].evaluate(namedSets, analysis, output); + if (oids == null) return null; + result.removeAll(oids); + } + + return result; + } +} + +class OrExpression extends SetExpression { + final List exprs; + + OrExpression(this.exprs); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final result = {}; + for (int i = 0; i < exprs.length; ++i) { + final oids = exprs[i].evaluate(namedSets, analysis, output); + if (oids == null) return null; + result.addAll(oids); + } + return result; + } +} + +class AndExpression extends SetExpression { + final SetExpression expr; + final List operands; + + AndExpression(this.expr, this.operands); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final nullableResult = expr.evaluate(namedSets, analysis, output)?.toSet(); + if (nullableResult == null) return null; + + Set result = nullableResult; + for (int i = 0; i < operands.length; ++i) { + final oids = operands[i].evaluate(namedSets, analysis, output); + if (oids == null) return null; + result = result.intersection(oids); + } + return result; + } +} + +class SampleExpression extends SetExpression { + static final _random = math.Random(); + + final SetExpression expr; + final int count; + + SampleExpression(this.expr, this.count); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = expr.evaluate(namedSets, analysis, output); + if (oids == null) return null; + + if (oids.isEmpty) return oids; + + final result = {}; + final l = oids.toList(); + while (result.length < count && result.length < oids.length) { + result.add(l[_random.nextInt(oids.length)]); + } + + return result; + } +} + +class ClosureExpression extends SetExpression { + final SetExpression expr; + final List patterns; + + ClosureExpression(this.expr, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final roots = expr.evaluate(namedSets, analysis, output); + if (roots == null) return null; + + final filter = analysis.parseTraverseFilter(patterns); + if (filter == null && + roots.length == analysis.roots.length && + roots.intersection(analysis.roots).length == roots.length) { + // The analysis needs to calculate the set of reachable objects + // already, so we re-use it instead of computing it again. + return analysis.reachableObjects; + } + return analysis.transitiveGraph(roots, filter); + } +} + +class UserClosureExpression extends SetExpression { + final SetExpression expr; + final List patterns; + + UserClosureExpression(this.expr, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final roots = expr.evaluate(namedSets, analysis, output); + if (roots == null) return null; + + final filter = analysis.parseTraverseFilter(patterns); + return analysis.reverseTransitiveGraph(roots, filter); + } +} + +class FollowExpression extends SetExpression { + final SetExpression objs; + final List patterns; + + FollowExpression(this.objs, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = objs.evaluate(namedSets, analysis, output); + if (oids == null) return null; + + return analysis.findReferences(oids, patterns); + } +} + +class UserFollowExpression extends SetExpression { + final SetExpression objs; + final List patterns; + + UserFollowExpression(this.objs, this.patterns); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = objs.evaluate(namedSets, analysis, output); + if (oids == null) return null; + + return analysis.findUsers(oids, patterns); + } +} + +class NamedExpression extends SetExpression { + final String name; + + NamedExpression(this.name); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = namedSets.getSet(name); + if (oids == null) { + output.printError('"$name" does not refer to a command or named set.'); + return null; + } + return oids; + } +} + +class SetNameExpression extends SetExpression { + final String name; + final SetExpression expr; + + SetNameExpression(this.name, this.expr); + + Set? evaluate(NamedSets namedSets, Analysis analysis, Output output) { + final oids = expr.evaluate(namedSets, analysis, output); + if (oids == null) return null; + + namedSets.nameSet(oids, name); + return oids; + } +} + +Set? parseAndEvaluate( + NamedSets namedSets, Analysis analysis, String text, Output output) { + final sexpr = parseExpression(text, output, namedSets.names.toSet()); + if (sexpr == null) return null; + return sexpr.evaluate(namedSets, analysis, output); +} + +SetExpression? parseExpression( + String text, Output output, Set namedSets) { + const help = 'See `help eval` for available expression types and arguments.'; + + final tokens = _TokenIterator(text); + final sexpr = parse(tokens, output, namedSets); + if (sexpr == null) { + output.printError(help); + return null; + } + if (tokens.moveNext()) { + tokens.movePrev(); + output.printError( + 'Found unexpected "${tokens.remaining}" after ${sexpr.runtimeType}.'); + output.printError(help); + return null; + } + return sexpr; +} + +final Map)> + parsingFunctions = { + // Filtering expressions. + 'filter': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final patterns = _parsePatterns(tokens, output); + return FilterExpression(s, patterns); + }, + 'dfilter': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final patterns = _parsePatterns(tokens, output); + return DFilterExpression(s, patterns); + }, + + // Traversing expressions. + 'follow': (_TokenIterator tokens, Output output, Set namedSets) { + final objs = parse(tokens, output, namedSets); + if (objs == null) return null; + final patterns = _parsePatterns(tokens, output); + return FollowExpression(objs, patterns); + }, + 'users': (_TokenIterator tokens, Output output, Set namedSets) { + final objs = parse(tokens, output, namedSets); + if (objs == null) return null; + final patterns = _parsePatterns(tokens, output); + return UserFollowExpression(objs, patterns); + }, + 'closure': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final patterns = _parsePatterns(tokens, output); + return ClosureExpression(s, patterns); + }, + 'uclosure': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final patterns = _parsePatterns(tokens, output); + return UserClosureExpression(s, patterns); + }, + + // Set operations + 'minus': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final operands = _parseExpressions(tokens, output, namedSets); + if (operands == null) return null; + return MinusExpression(s, operands); + }, + 'or': (_TokenIterator tokens, Output output, Set namedSets) { + final operands = _parseExpressions(tokens, output, namedSets); + if (operands == null) return null; + return OrExpression(operands); + }, + 'and': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + final operands = _parseExpressions(tokens, output, namedSets); + if (operands == null) return null; + return AndExpression(s, operands); + }, + + // Sample expression + 'sample': (_TokenIterator tokens, Output output, Set namedSets) { + final s = parse(tokens, output, namedSets); + if (s == null) return null; + + int count = 1; + if (tokens.moveNext()) { + if (tokens.current == ')') { + tokens.movePrev(); + } else { + tokens.current; + final value = int.tryParse(tokens.current); + if (value == null) { + output.printError( + '"sample" expression expects integer as 2nd argument.'); + return null; + } + count = value; + } + } + return SampleExpression(s, count); + }, + + // Sub-expression + '(': (_TokenIterator tokens, Output output, Set namedSets) { + final expr = parse(tokens, output, namedSets); + if (expr == null) return null; + + if (!tokens.moveNext()) { + output.printError('Expected closing ")" after "${tokens._text}".'); + return null; + } + if (tokens.current != ')') { + output.printError('Expected closing ")" but found "${tokens.current}".'); + tokens.movePrev(); + return null; + } + return expr; + }, +}; + +SetExpression? parse( + _TokenIterator tokens, Output output, Set namedSets) { + if (!tokens.moveNext()) { + output.printError('Reached end of input: expected expression'); + return null; + } + + final current = tokens.current; + final parserFun = parsingFunctions[current]; + if (parserFun != null) return parserFun(tokens, output, namedSets); + + if (current == ')') { + output.printError('Unexpected ).'); + return null; + } + if (tokens.moveNext()) { + if (tokens.current == '=') { + final expr = parse(tokens, output, namedSets); + if (expr == null) return null; + return SetNameExpression(current, expr); + } + tokens.movePrev(); + } + if (!namedSets.contains(current)) { + output.printError('There is no set with name "$current". See `info`.'); + return null; + } + return NamedExpression(current); +} + +class _TokenIterator { + final String _text; + + String? _current = null; + int _index = 0; + + _TokenIterator(this._text); + + String get current => _current!; + + bool get isAtEnd => _index == _text.length; + + bool moveNextPattern() { + _current = null; + + int start = _index; + while ( + start < _text.length && _text.codeUnitAt(start) == ' '.codeUnitAt(0)) { + start++; + } + if (start == _text.length) return false; + + int openCount = 0; + int end = start; + while (end < _text.length) { + final char = _text.codeUnitAt(end); + + if (char == '('.codeUnitAt(0)) { + openCount++; + end++; + continue; + } + if (char == ')'.codeUnitAt(0)) { + openCount--; + if (openCount >= 0) { + end++; + continue; + } + // This ) has no corresponding (. + if (start == end) return false; + _current = _text.substring(start, end); + _index = end; + return true; + } + if (char == ' '.codeUnitAt(0)) { + _current = _text.substring(start, end); + _index = end; + return true; + } + + end++; + } + + _current = _text.substring(start, end); + _index = end; + return true; + } + + bool moveNext() { + int start = _index; + while ( + start < _text.length && _text.codeUnitAt(start) == ' '.codeUnitAt(0)) { + start++; + } + if (start == _text.length) return false; + + int end = start + 1; + + final firstChar = _text.codeUnitAt(start); + if (firstChar == '('.codeUnitAt(0) || firstChar == ')'.codeUnitAt(0)) { + _current = _text.substring(start, end); + _index = end; + return true; + } + if (firstChar == '=') { + _current = _text.substring(start, end); + _index = end; + return true; + } + + while (end < _text.length && _text.codeUnitAt(end) != ' '.codeUnitAt(0)) { + final char = _text.codeUnitAt(end); + if (char == '('.codeUnitAt(0) || char == ')'.codeUnitAt(0)) { + _current = _text.substring(start, end); + _index = end; + return true; + } + if (char == '=') { + _current = _text.substring(start, end); + _index = end; + return true; + } + end++; + } + + _current = _text.substring(start, end); + _index = end; + return true; + } + + void movePrev() { + _index -= current.length; + _current = null; + } + + String? peek() { + if (!moveNext()) return null; + final peek = current; + movePrev(); + return peek; + } + + String get remaining => _text.substring(_index); +} + +List? _parseExpressions( + _TokenIterator tokens, Output output, Set namedSets) { + final all = []; + + while (true) { + final peek = tokens.peek(); + if (peek == null || peek == ')') break; + final e = parse(tokens, output, namedSets); + if (e == null) return null; + all.add(e); + } + return all; +} + +List _parsePatterns(_TokenIterator tokens, Output output) { + final patterns = []; + while (tokens.moveNextPattern()) { + if (tokens.current == ')') { + tokens.movePrev(); + } + patterns.add(tokens.current); + } + return patterns; +} + +class NamedSets { + final Map> _namedSets = {}; + int _varIndex = 0; + + List get names => _namedSets.keys.toList(); + + String nameSet(Set oids, [String? id]) { + id ??= _generateName(); + _namedSets[id] = oids; + return id; + } + + Set? getSet(String name) => _namedSets[name]; + + bool hasSetName(String name) => _namedSets.containsKey(name); + + void clear(String name) { + _namedSets.remove(name); + } + + void clearWhere(bool Function(String) cond) { + _namedSets.removeWhere((name, _) => cond(name)); + } + + void forEach(void Function(String, Set) fun) { + _namedSets.forEach(fun); + } + + String _generateName() => '\$${_varIndex++}'; +} + +abstract class Output { + void print(String message) {} + void printError(String message) {} +} + +const dslDescription = ''' +An `` can be + +Filtering a set of objects based on class/field or data: + + filter $dslFilter + dfilter [{<,<=,==,>=,>}len]* [content-pattern]* + +Traversing to references or uses of the set of objects: + + follow $dslFilter + users $dslFilter + closure $dslFilter + uclosure $dslFilter + +Performing a set operation on multiple sets: + + or * + and * + sub * + +Sample a random element from a set: + + sample ? + +Name a set of objects or retrieve the objects for a given name: + + + = +'''; + +const dslFilter = '[class-pattern]* [class-pattern:field-pattern]*'; diff --git a/runtime/tools/heapsnapshot/lib/src/format.dart b/runtime/tools/heapsnapshot/lib/src/format.dart new file mode 100644 index 00000000000..dfead236595 --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/src/format.dart @@ -0,0 +1,204 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:typed_data'; + +import 'package:vm_service/vm_service.dart'; + +import 'analysis.dart'; + +String format(int a) => a.toString().padLeft(6, ' '); +String formatBytes(int a) => (a ~/ 1024).toString().padLeft(6, ' ') + ' kb'; +String truncateString(String s) { + int index; + + index = s.indexOf('\n'); + if (index >= 0) s = s.substring(index); + + index = s.indexOf('\r'); + if (index >= 0) s = s.substring(index); + + if (s.length > 30) s = s.substring(30); + return s; +} + +String formatHeapStats(HeapStats stats, {int? maxLines, int? sizeCutoff}) { + assert(sizeCutoff == null || sizeCutoff >= 0); + assert(maxLines == null || maxLines >= 0); + + final table = Table(); + table.addRow(['size', 'count', 'class']); + table.addRow(['--------', '--------', '--------']); + int totalSize = 0; + int totalCount = 0; + for (int i = 0; i < stats.classes.length; ++i) { + final c = stats.classes[i]; + final count = stats.counts[c.classId]; + final size = stats.sizes[c.classId]; + + totalSize += size; + totalCount += count; + + if (sizeCutoff == null || size >= sizeCutoff) { + if (maxLines == null || i < maxLines) { + table.addRow( + [formatBytes(size), format(count), '${c.name} ${c.libraryUri}']); + } + } + } + if (table.rows > 3) { + table.addRow(['--------', '--------']); + table.addRow([formatBytes(totalSize), format(totalCount)]); + } + return table.asString; +} + +String formatDataStats(HeapDataStats stats, {int? maxLines, int? sizeCutoff}) { + assert(sizeCutoff == null || sizeCutoff >= 0); + assert(maxLines == null || maxLines >= 0); + + final table = Table(); + table.addRow(['size', 'unique-size', 'count', 'class', 'data']); + table.addRow(['--------', '--------', '--------', '--------', '--------']); + + int totalSize = 0; + int totalUniqueSize = 0; + int totalCount = 0; + + final List datas = stats.datas; + for (int i = 0; i < datas.length; ++i) { + final data = datas[i]; + + totalSize += data.size; + totalUniqueSize += data.totalSize; + totalCount += data.count; + + if (sizeCutoff == null || data.totalSize >= sizeCutoff) { + if (maxLines == null || i < maxLines) { + table.addRow([ + formatBytes(data.totalSize), + formatBytes(data.size), + format(data.count), + data.klass, + data.valueAsString, + ]); + } + } + } + if (table.rows > 3) { + table.addRow(['--------', '--------', '--------']); + table.addRow([ + formatBytes(totalUniqueSize), + formatBytes(totalSize), + format(totalCount) + ]); + } + return table.asString; +} + +String formatRetainingPath(HeapSnapshotGraph graph, DedupedUint32List rpath) { + final path = _stringifyRetainingPath(graph, rpath); + final bool wasTruncated = rpath.path.last != /*root*/ 1; + final sb = StringBuffer(); + for (int i = 0; i < path.length; ++i) { + final indent = i >= 2 ? (i - 1) : 0; + sb.writeln(' ' * 4 * indent + (i == 0 ? '' : '⮑ ') + '${path[i]}'); + } + if (wasTruncated) { + sb.writeln(' ' * 4 * (path.length - 1) + '⮑ …'); + } + return sb.toString(); +} + +String formatDominatorPath(HeapSnapshotGraph graph, DedupedUint32List dpath) { + final path = _stringifyDominatorPath(graph, dpath); + final bool wasTruncated = dpath.path.last != /*root*/ 1; + final sb = StringBuffer(); + for (int i = 0; i < path.length; ++i) { + final indent = i >= 2 ? (i - 1) : 0; + sb.writeln(' ' * 4 * indent + (i == 0 ? '' : '⮑ ') + '${path[i]}'); + } + if (wasTruncated) { + sb.writeln(' ' * 4 * (path.length - 1) + '⮑ …'); + } + return sb.toString(); +} + +List _stringifyRetainingPath( + HeapSnapshotGraph graph, DedupedUint32List rpath) { + final path = rpath.path; + final spath = []; + for (int i = 0; i < path.length; i += 2) { + final klass = graph.classes[path[i]]; + + String? fieldName; + String prefix = ''; + if (i > 0) { + final int value = path[i - 1]; + final hasUniqueOwner = (value & (1 << 0)) == 1; + final fieldIndex = value >> 1; + if (fieldIndex != DedupedUint32List.noFieldIndex) { + final field = klass.fields[fieldIndex]; + assert(field.index == fieldIndex); + fieldName = field.name; + } + prefix = (hasUniqueOwner ? '・' : '﹢'); + } + + spath.add(prefix + + '${klass.name}' + + (fieldName != null ? '.$fieldName' : '') + + ' (${klass.libraryUri})'); + } + return spath; +} + +List _stringifyDominatorPath( + HeapSnapshotGraph graph, DedupedUint32List rpath) { + final path = rpath.path; + final spath = []; + for (int i = 0; i < path.length; i++) { + final klass = graph.classes[path[i]]; + spath.add('${klass.name} (${klass.libraryUri})'); + } + return spath; +} + +class Table { + final List> _rows = []; + int _maxColumn = -1; + + int get rows => _rows.length; + + void addRow(List row) { + _maxColumn = row.length > _maxColumn ? row.length : _maxColumn; + _rows.add(row); + } + + String get asString { + if (_rows.isEmpty) return ''; + + final colSizes = Uint32List(_maxColumn); + for (final row in _rows) { + for (int i = 0; i < row.length; ++i) { + final value = row[i]; + final c = colSizes[i]; + if (value.length > c) colSizes[i] = value.length; + } + } + + final sb = StringBuffer(); + for (final row in _rows) { + for (int i = 0; i < row.length; ++i) { + row[i] = row[i].padRight(colSizes[i], ' '); + } + sb.writeln(row.join(' ')); + } + return sb.toString().trimRight(); + } +} + +String indent(String left, String text) { + return left + text.replaceAll('\n', '\n$left'); +} diff --git a/runtime/tools/heapsnapshot/lib/src/load.dart b/runtime/tools/heapsnapshot/lib/src/load.dart new file mode 100644 index 00000000000..36aa920ffa2 --- /dev/null +++ b/runtime/tools/heapsnapshot/lib/src/load.dart @@ -0,0 +1,48 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:typed_data'; + +import 'package:vm_service/vm_service.dart'; +import 'package:vm_service/vm_service_io.dart'; + +Future> loadFromUri(Uri uri) async { + final wsUri = uri.replace(scheme: 'ws', path: '/ws'); + final service = await vmServiceConnectUri(wsUri.toString()); + try { + final r = await _getHeapsnapshot(service); + return r; + } finally { + await service.dispose(); + } +} + +Future> _getHeapsnapshot(VmService service) async { + final vm = await service.getVM(); + final vmIsolates = vm.isolates!; + if (vmIsolates.isEmpty) { + throw 'Could not find first isolate (expected it to be running already)'; + } + final isolateRef = vmIsolates.first; + + await service.streamListen(EventStreams.kHeapSnapshot); + + final chunks = []; + final done = Completer(); + late StreamSubscription streamSubscription; + streamSubscription = service.onHeapSnapshotEvent.listen((e) async { + chunks.add(e.data!); + if (e.last!) { + await service.streamCancel(EventStreams.kHeapSnapshot); + await streamSubscription.cancel(); + done.complete(); + } + }); + + await service.requestHeapSnapshot(isolateRef.id!); + await done.future; + + return chunks; +} diff --git a/runtime/tools/heapsnapshot/pubspec.yaml b/runtime/tools/heapsnapshot/pubspec.yaml new file mode 100644 index 00000000000..46d3584c00c --- /dev/null +++ b/runtime/tools/heapsnapshot/pubspec.yaml @@ -0,0 +1,24 @@ +name: heapsnapshot +version: 0.1.0 +description: Utilities for analysing heapsnapshots generated by Dart VM. +repository: https://github.com/dart-lang/sdk/tree/main/pkg/heapsnapshot +# This package is right now not intended to be published. +# See also https://github.com/dart-lang/sdk/issues/50061 +publish_to: none + +environment: + sdk: '>=2.12.0 <3.0.0' + +dependencies: + args: ^2.0.0 + vm_service: ^9.3.0 + +dev_dependencies: + test: ^1.21.6 + path: ^1.8.0 + dart_console: ^1.1.2 + +dependency_overrides: + # For obj.referrers (TODO: publish new package:vm_service version) + vm_service: + path: ../../../pkg/vm_service diff --git a/runtime/tools/heapsnapshot/test/cli_test.dart b/runtime/tools/heapsnapshot/test/cli_test.dart new file mode 100644 index 00000000000..04fcfabab4a --- /dev/null +++ b/runtime/tools/heapsnapshot/test/cli_test.dart @@ -0,0 +1,354 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:heapsnapshot/src/cli.dart'; +import 'package:path/path.dart' as path; + +import 'package:test/test.dart'; + +import 'utils.dart'; + +class ErrorCollector extends Output { + final errors = []; + final output = []; + final all = []; + + void printError(String error) { + errors.add(error); + all.add(error); + } + + void print(String message) { + output.add(message); + all.add(message); + } + + void clear() { + errors.clear(); + output.clear(); + all.clear(); + } + + String get log => all.join('\n'); +} + +main([List args = const []]) { + if (!args.isEmpty) { + // We're in the child. + if (args.single != '--child') throw 'failed'; + + // Force initialize of the data we want in the heapsnapshot. + print(global.use); + print(weakTest.use); + print('Child ready'); + return; + } + + group('cli', () { + late Testee testee; + late String testeeUrl; + late Directory snapshotDir; + late String heapsnapshotFile; + late ErrorCollector errorCollector; + late CliState cliState; + + setUpAll(() async { + snapshotDir = Directory.systemTemp.createTempSync('snapshot'); + heapsnapshotFile = path.join(snapshotDir.path, 'current.heapsnapshot'); + + testee = Testee('test/cli_test.dart'); + testeeUrl = await testee.start(['--child']); + await testee.getHeapsnapshotAndWriteTo(heapsnapshotFile); + + errorCollector = ErrorCollector(); + cliState = CliState(errorCollector); + }); + + tearDownAll(() async { + snapshotDir.deleteSync(recursive: true); + await testee.close(); + }); + + late String log; + Future run(String commandString) async { + final args = commandString.split(' ').where((p) => !p.isEmpty).toList(); + print('-----------------------'); + print('Running: $commandString'); + await cliCommandRunner.run(cliState, args); + log = errorCollector.log; + print(log.trim()); + errorCollector.clear(); + } + + expectLog(String expected) { + expect(log.trim(), expected.trim()); + } + + expectLogPattern(String pattern) { + final logLines = log + .split('\n') + .map((p) => p.trim()) + .where((p) => !p.isEmpty) + .toList(); + final patternLines = pattern + .split('\n') + .map((p) => p.trim()) + .where((p) => !p.isEmpty) + .toList(); + if (logLines.length != patternLines.length) { + print('Expected pattern:'); + print(' ' + patternLines.join('\n ')); + print('But got:'); + print(' ' + logLines.join('\n ')); + } + for (int i = 0; i < logLines.length; ++i) { + final log = logLines[i]; + final pattern = patternLines[i]; + if (!RegExp(pattern).hasMatch(log)) { + print('[$i] $log does not match pattern "$pattern"'); + expect(false, true); + } + } + } + + const sp = r'\{#\d+\}'; + + test('cli commands', () async { + // Test loading from Uri & search for the Global object. + await run('load $testeeUrl'); + expectLog('Loaded heapsnapshot from "$testeeUrl".'); + + await run('stat filter (closure roots) Global'); + expectLogPattern(r''' + size count class + -------- -------- -------- + 0 kb 1 Global .*/cli_test.dart + '''); + + // Test loading from file & do remaining tests. + + await run('load $heapsnapshotFile'); + expectLog('Loaded heapsnapshot from "$heapsnapshotFile".'); + + await run('info'); + expectLogPattern(''' +Known named sets: + roots $sp + '''); + + await run('eval all = closure roots'); + expectLogPattern('all $sp'); + + await run('info'); + expectLogPattern(''' +Known named sets: + roots $sp + all $sp + '''); + + await run('global = filter all Global'); + expectLogPattern(r'global \{#1\}'); + + await run('stats global'); + expectLogPattern(''' +size *count *class +-------- *-------- *-------- +0 kb *1 *Global .*cli_test.dart + '''); + + await run('dstats -c filter (closure global) String'); + expectLogPattern(''' +size unique-size count class data +-------- -------- -------- -------- -------- + 0 kb 0 kb 4 _OneByteString #nonSharedString# + 0 kb 0 kb 2 _OneByteString #barUniqueString + 0 kb 0 kb 2 _OneByteString #fooUniqueString + 0 kb 0 kb 1 _OneByteString #sharedString +-------- -------- -------- + 0 kb 0 kb 9 + '''); + + await run('lists = filter (closure global) _List'); + expectLogPattern('lists $sp'); + + await run('dstats -c lists'); + expectLogPattern(''' +size unique-size count class data +-------- -------- -------- -------- -------- + 0 kb 0 kb 1 _List len:2 + 0 kb 0 kb 1 _List len:2 + 0 kb 0 kb 1 _List len:1 + 0 kb 0 kb 1 _List len:1 + 0 kb 0 kb 1 _List len:0 + 0 kb 0 kb 1 _List len:0 +-------- -------- -------- + 0 kb 0 kb 6 + '''); + + await run( + 'stats foobar = (follow (follow global) ^:type_arguments ^Root)'); + expectLogPattern(''' +size count class +-------- -------- -------- + 0 kb 2 Foo .*cli_test.dart + 0 kb 2 Bar .*cli_test.dart +-------- -------- + 0 kb 4 + '''); + + await run('examine users foobar'); + expectLogPattern(r''' + _List@\d+ .* { + type_arguments_ + length_ + \[0\] *Foo@\d+ .*/cli_test.dart + \[1\] *Foo@\d+ .*/cli_test.dart + } + _List@\d+ .* { + type_arguments_ + length_ + \[0\] *Bar@\d+ .*/cli_test.dart + \[1\] *Bar@\d+ .*/cli_test.dart + } + '''); + + await run('examine users users foobar'); + expectLogPattern(r''' +Global@\d+ .*/cli_test.dart.* { + foos _List@\d+ + bars _List@\d+ +} + '''); + + await run('users (follow global :bars)'); + + await run('retainers -n10 filter (closure global) String'); + + expectLogPattern(r''' +There are 2 retaining paths of +_OneByteString +⮑ ・Bar.barLocal .*/cli_test.dart + ⮑ ・_List + ⮑ ・Global.bars .*/cli_test.dart + ⮑ ・Isolate.global + ⮑ ・Root + + +There are 2 retaining paths of +_OneByteString +⮑ ・Bar.barUnique .*/cli_test.dart + ⮑ ・_List + ⮑ ・Global.bars .*/cli_test.dart + ⮑ ・Isolate.global + ⮑ ・Root + + +There are 2 retaining paths of +_OneByteString +⮑ ・Foo.fooLocal .*/cli_test.dart + ⮑ ・_List + ⮑ ・Global.foos .*/cli_test.dart + ⮑ ・Isolate.global + ⮑ ・Root + + +There are 2 retaining paths of +_OneByteString +⮑ ・Foo.fooUnique .*/cli_test.dart + ⮑ ・_List + ⮑ ・Global.foos .*/cli_test.dart + ⮑ ・Isolate.global + ⮑ ・Root + + +There are 1 retaining paths of +_OneByteString +⮑ ﹢Foo.fooShared .*/cli_test.dart + ⮑ ・_List + ⮑ ・Global.foos .*/cli_test.dart + ⮑ ・Isolate.global + ⮑ ・Root + '''); + + await run('describe-filter Foo:List ^Bar'); + + expectLogPattern(r''' +The traverse filter expression "Foo:List \^Bar" matches: + +\[\-\] Bar +\[ \] Foo +\[\+\] \.fooList0 + '''); + + await run('weakly-held-object = follow (filter all WeakTest) :object'); + await run('stats uclosure weakly-held-object'); + + expectLogPattern(r''' +size count class +-------- -------- -------- +0 kb 1 WeakTest .*/cli_test.dart +0 kb 1 Object dart:core +0 kb 1 Root +0 kb 1 Isolate +-------- -------- +0 kb 4 + '''); + }); + }); +} + +final global = Global(); + +var marker = '|'; +var sharedString = 'x'; + +class Foo { + final fooShared = sharedString; + final fooLocal = marker + 'nonSharedString' + marker; + final fooUnique = marker + 'fooUniqueString'; + final fooList0 = List.filled(0, null); + + String get use => 'Foo($fooShared, $fooLocal, $fooUnique, $fooList0)'; +} + +class Bar { + final barShared = sharedString; + final barLocal = marker + 'nonSharedString' + marker; + final barUnique = marker + 'barUniqueString'; + final barList1 = List.filled(1, null); + + String get use => 'Bar($barShared, $barLocal, $barUnique, $barList1)'; +} + +class Global { + late final foos; + late final bars; + + Global() { + marker = '#'; + sharedString = marker + 'sharedString'; + foos = [Foo(), Foo()].toList(growable: false); + bars = [Bar(), Bar()].toList(growable: false); + sharedString = ''; + } + + String get use => + '${foos.map((l) => l.use).toList()}|${bars.map((l) => l.use).toList()}|$sharedString'; +} + +final weakTest = WeakTest(Object()); + +class WeakTest { + final Object object; + final List> weakList; + final Finalizer finalizer; + + WeakTest(this.object) + : weakList = List.filled(1, WeakReference(object)), + finalizer = Finalizer((_) {})..attach(object, Object(), detach: object); + + String get use => '$object|$weakList|$finalizer'; +} diff --git a/runtime/tools/heapsnapshot/test/expression_test.dart b/runtime/tools/heapsnapshot/test/expression_test.dart new file mode 100644 index 00000000000..8e3c82a5ef2 --- /dev/null +++ b/runtime/tools/heapsnapshot/test/expression_test.dart @@ -0,0 +1,202 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:heapsnapshot/src/expression.dart'; + +import 'package:test/test.dart'; + +class ErrorCollector extends Output { + final errors = []; + void printError(String error) { + errors.add(error); + } + + void print(String message) {} +} + +main([List args = const []]) { + group('parser', () { + late ErrorCollector ec; + + setUp(() { + ec = ErrorCollector(); + }); + + void match(SetExpression? expr, void Function(T) fun) { + expect(expr is T, true); + fun(expr as T); + } + + void matchNamed(SetExpression? expr, String name) { + match(expr, (expr) { + expect(expr.name, name); + }); + } + + void parseMatch(String input, void Function(T) fun) { + final expr = + parseExpression(input, ec, {'all', 'set1', 'set2', 'set3', 'set4'}); + match(expr, fun); + } + + void parseError(String input, Set namedSets, List errors) { + final expr = parseExpression(input, ec, namedSets); + expect(expr, null); + expect(ec.errors, errors); + } + + group('expression', () { + test('filter', () { + parseMatch('filter all cls (cls2:field)', (expr) { + expect(expr.patterns, ['cls', '(cls2:field)']); + matchNamed(expr.expr, 'all'); + }); + }); + + test('dfilter', () { + parseMatch('dfilter all content ==0', (expr) { + expect(expr.patterns, ['content', '==0']); + matchNamed(expr.expr, 'all'); + }); + }); + test('minus', () { + parseMatch('minus set1 set2 set3', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.operands.length, 2); + matchNamed(expr.operands[0], 'set2'); + matchNamed(expr.operands[1], 'set3'); + }); + }); + test('or', () { + parseMatch('or set1 set2 set3', (expr) { + expect(expr.exprs.length, 3); + matchNamed(expr.exprs[0], 'set1'); + matchNamed(expr.exprs[1], 'set2'); + matchNamed(expr.exprs[2], 'set3'); + }); + }); + test('or-empty', () { + parseMatch('or', (expr) { + expect(expr.exprs.length, 0); + }); + }); + test('and', () { + parseMatch('and set1 set2 set3', (expr) { + expect(expr.operands.length, 2); + matchNamed(expr.expr, 'set1'); + matchNamed(expr.operands[0], 'set2'); + matchNamed(expr.operands[1], 'set3'); + }); + }); + test('sample', () { + parseMatch('sample set1', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.count, 1); + }); + }); + + test('sample-num', () { + parseMatch('sample set1 10', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.count, 10); + }); + }); + + test('closure', () { + parseMatch('closure set1', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.patterns, []); + }); + }); + test('closure-filter', () { + parseMatch('closure set1 cls cls:field', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.patterns, ['cls', 'cls:field']); + }); + }); + + test('uclosure', () { + parseMatch('uclosure set1', (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.patterns, []); + }); + }); + test('uclosure-filter', () { + parseMatch('uclosure set1 cls cls:field', + (expr) { + matchNamed(expr.expr, 'set1'); + expect(expr.patterns, ['cls', 'cls:field']); + }); + }); + + test('follow', () { + parseMatch('follow set1 cls cls:field', (expr) { + matchNamed(expr.objs, 'set1'); + expect(expr.patterns, ['cls', 'cls:field']); + }); + }); + test('users', () { + parseMatch('users set1 cls cls:field', (expr) { + matchNamed(expr.objs, 'set1'); + expect(expr.patterns, ['cls', 'cls:field']); + }); + }); + + test('set-name', () { + parseMatch('set1 = closure set1', (expr) { + match(expr.expr, (expr) {}); + expect(expr.name, 'set1'); + }); + }); + + test('parens', () { + parseMatch('or (( set1 )) ( set2 ) ( set3) (set4 )', + (expr) { + expect(expr.exprs.length, 4); + matchNamed(expr.exprs[0], 'set1'); + matchNamed(expr.exprs[1], 'set2'); + matchNamed(expr.exprs[2], 'set3'); + matchNamed(expr.exprs[3], 'set4'); + }); + }); + }); + + group('expression-errors', () { + test('empty-and', () { + parseError('and', {}, [ + 'Reached end of input: expected expression', + 'See `help eval` for available expression types and arguments.' + ]); + }); + test('empty-minus', () { + parseError('minus', {}, [ + 'Reached end of input: expected expression', + 'See `help eval` for available expression types and arguments.' + ]); + }); + test('unknown set', () { + parseError('closure foobar', {}, [ + 'There is no set with name "foobar". See `info`.', + 'See `help eval` for available expression types and arguments.' + ]); + }); + test('missing )', () { + parseError('closure (a', { + 'a' + }, [ + 'Expected closing ")" after "closure (a".', + 'See `help eval` for available expression types and arguments.' + ]); + }); + test('garbage', () { + parseError('sample set1 10 foo', { + 'set1' + }, [ + 'Found unexpected "foo" after SampleExpression.', + 'See `help eval` for available expression types and arguments.' + ]); + }); + }); + }); +} diff --git a/runtime/tools/heapsnapshot/test/utils.dart b/runtime/tools/heapsnapshot/test/utils.dart new file mode 100644 index 00000000000..2bb68894536 --- /dev/null +++ b/runtime/tools/heapsnapshot/test/utils.dart @@ -0,0 +1,99 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' hide BytesBuilder; +import 'dart:typed_data' show BytesBuilder; + +import 'package:heapsnapshot/src/load.dart'; +import 'package:vm_service/vm_service_io.dart'; + +class Testee { + final String testFile; + + late final Process _process; + final Completer _serviceUri = Completer(); + + Testee(this.testFile); + + Future start(List args) async { + var script = Platform.script.toFilePath(); + if (script.endsWith('dart_2.dill')) { + // We run via `dart test` and the `package:test` has wrapped the `main()` + // function. We don't want to invoke the wrapper as subprocess, but rather + // the actual file. + script = testFile; + } + + final processArgs = [ + ...Platform.executableArguments, + '--disable-dart-dev', + '--disable-service-auth-codes', + '--enable-vm-service:0', + '--pause-isolates-on-exit', + script, + ...args, + ]; + _process = await Process.start(Platform.executable, processArgs); + final childReadyCompleter = Completer(); + _process.stdout + .transform(Utf8Decoder()) + .transform(const LineSplitter()) + .listen((line) { + print('child-stdout: $line'); + final urlStart = line.indexOf('http://'); + if (line.contains('http')) { + _serviceUri.complete(line.substring(urlStart).trim()); + return; + } + if (line.contains('Child ready')) { + childReadyCompleter.complete(); + return; + } + }); + _process.stderr + .transform(Utf8Decoder()) + .transform(const LineSplitter()) + .listen((line) { + print('child-stderr: $line'); + }); + final uri = await _serviceUri.future; + await childReadyCompleter.future; + return uri; + } + + Future getHeapsnapshotAndWriteTo(String filename) async { + final chunks = await loadFromUri(Uri.parse(await _serviceUri.future)); + final bytesBuilder = BytesBuilder(); + for (final bd in chunks) { + bytesBuilder + .add(bd.buffer.asUint8List(bd.offsetInBytes, bd.lengthInBytes)); + } + final bytes = bytesBuilder.toBytes(); + + File(filename).writeAsBytesSync(bytes); + } + + Future close() async { + final wsUri = + Uri.parse(await _serviceUri.future).replace(scheme: 'ws', path: '/ws'); + final service = await vmServiceConnectUri(wsUri.toString()); + + final vm = await service.getVM(); + final vmIsolates = vm.isolates!; + if (vmIsolates.isEmpty) { + throw 'Could not find first isolate (expected it to be running already)'; + } + final isolateRef = vmIsolates.first; + + // The isolate is hanging on --pause-on-exit. + await service.resume(isolateRef.id!); + final exitCode = await _process.exitCode; + print('child-exitcode: $exitCode'); + if (exitCode != 0) { + throw 'Child process terminated unsucessfully'; + } + } +} diff --git a/tools/generate_package_config.dart b/tools/generate_package_config.dart index 1a8e083b5a7..bb2649726f2 100644 --- a/tools/generate_package_config.dart +++ b/tools/generate_package_config.dart @@ -23,6 +23,7 @@ void main(List args) { platform('runtime/observatory'), platform('runtime/observatory/tests/service/observatory_test_package'), platform('runtime/observatory_2'), + platform('runtime/tools/heapsnapshot'), platform('sdk/lib/_internal/sdk_library_metadata'), platform('third_party/devtools/devtools_shared'), platform('tools/package_deps'),