From 23b1f38981f70f8dd44d6a7aafa3824401b5ae3c Mon Sep 17 00:00:00 2001 From: Chris Evans Date: Wed, 9 Mar 2022 14:17:44 +0000 Subject: [PATCH] Fixes for reland for analyze_snapshot program 1. Disable copying of executable for SDK binary signing 2. Fixes for ASAN memory leak in executable TEST=runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart This is a reland of 5d7d8a377dec742bd388849d139dbabdf9bee767 Original change's description: > Reland "Reland "[vm] Add analyze_snapshot tool for AOT snapshot inspection"" > > This is a reland of 8d1eedca64d29e0e25d4536e4b145bf584c83eed > > Disable builds for Fuchsia > > TEST=runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart > Original change's description: > > Reland "[vm] Add analyze_snapshot tool for AOT snapshot inspection" > > > > This is a reland of 19e57493088ac57c1f3f45918b9dc97d15de42e0 > > > > TEST=runtime/tests/vm/dart/analyze_snapshot_binary_test.dart > > > > Original change's description: > > > [vm] Add analyze_snapshot tool for AOT snapshot inspection > > > > > > Current skeleton to allow for instrumentation snapshots that can be > > > built alongside Dart compilation artifacts and easily referenced for > > > specific versions between Snapshot hash <-> DartSDK <-> Flutter Engine > > > > > > TEST=runtime/tests/vm/dart/analyze_snapshot_binary_test.dart > > > > > > Change-Id: Ie3757a265bbf457506c72fb62a625fea7bedcb68 > > > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/221087 > > > Reviewed-by: Slava Egorov > > > Commit-Queue: Slava Egorov > > > > Change-Id: Ia1ea0071d30818440ae48484ff6c406236af5a4e > > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/224526 > > Reviewed-by: Slava Egorov > > Commit-Queue: Slava Egorov > > Change-Id: I769ced4cbe6eb926b8df36a15ca13c3145632082 > Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/233890 > Reviewed-by: Slava Egorov > Commit-Queue: Chris Evans Change-Id: I1e6e57dda56f1710cc3a52e35d4067910930a701 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/234500 Reviewed-by: Slava Egorov Reviewed-by: Chris Evans Commit-Queue: Chris Evans --- BUILD.gn | 11 + runtime/BUILD.gn | 3 + runtime/bin/BUILD.gn | 39 +++ runtime/bin/analyze_snapshot.cc | 262 ++++++++++++++++++ runtime/include/analyze_snapshot_api.h | 27 ++ .../vm/dart/analyze_snapshot_binary_test.dart | 214 ++++++++++++++ .../dart_2/analyze_snapshot_binary_test.dart | 212 ++++++++++++++ runtime/tests/vm/vm.status | 7 + runtime/vm/analyze_snapshot_api_impl.cc | 202 ++++++++++++++ sdk/BUILD.gn | 1 + tools/gn.py | 2 + 11 files changed, 980 insertions(+) create mode 100644 runtime/bin/analyze_snapshot.cc create mode 100644 runtime/include/analyze_snapshot_api.h create mode 100644 runtime/tests/vm/dart/analyze_snapshot_binary_test.dart create mode 100644 runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart create mode 100644 runtime/vm/analyze_snapshot_api_impl.cc diff --git a/BUILD.gn b/BUILD.gn index ad09ed86c0b..6295280a080 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -39,6 +39,7 @@ group("runtime") { # Fuchsia has run_vm_tests marked testonly. testonly = true } + deps = [ "runtime/bin:dart", "runtime/bin:entrypoints_verification_test", @@ -59,6 +60,16 @@ group("runtime") { ] } + # We do not support AOT on ia32 and should therefore cannot provide native + # snapshot tooling. + if (dart_target_arch != "ia32") { + if (dart_runtime_mode == "release") { + deps += [ "runtime/bin:analyze_snapshot_product" ] + } else { + deps += [ "runtime/bin:analyze_snapshot" ] + } + } + if (is_linux || is_android) { deps += [ "runtime/bin:abstract_socket_test" ] } diff --git a/runtime/BUILD.gn b/runtime/BUILD.gn index a4c477afc30..12d3dfb0c40 100644 --- a/runtime/BUILD.gn +++ b/runtime/BUILD.gn @@ -242,6 +242,7 @@ config("dart_libfuzzer_config") { source_set("dart_api") { public_configs = [ ":dart_public_config" ] sources = [ + "include/analyze_snapshot_api.h", "include/dart_api.h", "include/dart_api_dl.c", "include/dart_api_dl.h", @@ -289,9 +290,11 @@ library_for_all_configs("libdart") { public_configs = [ ":dart_public_config" ] sources = [ "$target_gen_dir/version.cc", + "include/analyze_snapshot_api.h", "include/dart_api.h", "include/dart_native_api.h", "include/dart_tools_api.h", + "vm/analyze_snapshot_api_impl.cc", "vm/dart_api_impl.cc", "vm/native_api_impl.cc", "vm/version.h", diff --git a/runtime/bin/BUILD.gn b/runtime/bin/BUILD.gn index ea7d40dc6a5..04e73a2c3a3 100644 --- a/runtime/bin/BUILD.gn +++ b/runtime/bin/BUILD.gn @@ -880,6 +880,45 @@ dart_executable("dart_precompiled_runtime_product") { extra_deps += [ ":elf_loader_product" ] } +dart_executable("analyze_snapshot") { + extra_configs = [ "..:dart_precompiled_runtime_config" ] + extra_deps = [ + "..:libdart_precompiled_runtime", + "../platform:libdart_platform_precompiled_runtime", + ] + + extra_sources = [ + "analyze_snapshot.cc", + "builtin.cc", + "loader.cc", + "loader.h", + ] + + if (dart_runtime_mode == "release") { + extra_deps += [ ":elf_loader_product" ] + } else { + extra_deps += [ ":elf_loader" ] + } +} + +dart_executable("analyze_snapshot_product") { + use_product_mode = true + extra_configs = [ "..:dart_precompiled_runtime_config" ] + extra_deps = [ + "..:libdart_precompiled_runtime_product", + "../platform:libdart_platform_precompiled_runtime_product", + ] + + extra_sources = [ + "analyze_snapshot.cc", + "builtin.cc", + "loader.cc", + "loader.h", + ] + + extra_deps += [ ":elf_loader_product" ] +} + executable("process_test") { sources = [ "process_test.cc" ] } diff --git a/runtime/bin/analyze_snapshot.cc b/runtime/bin/analyze_snapshot.cc new file mode 100644 index 00000000000..1eb4c14c972 --- /dev/null +++ b/runtime/bin/analyze_snapshot.cc @@ -0,0 +1,262 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "bin/elf_loader.h" +#include "bin/error_exit.h" +#include "bin/file.h" + +#include "bin/options.h" +#include "bin/platform.h" + +#if !defined(DART_HOST_OS_FUCHSIA) +#include "include/analyze_snapshot_api.h" +#endif + +namespace dart { +namespace bin { +#if !defined(DART_HOST_OS_FUCHSIA) + +#define STRING_OPTIONS_LIST(V) V(out, out_path) + +#define BOOL_OPTIONS_LIST(V) \ + V(help, help) \ + V(version, version) + +#define STRING_OPTION_DEFINITION(flag, variable) \ + static const char* variable = nullptr; \ + DEFINE_STRING_OPTION(flag, variable) +STRING_OPTIONS_LIST(STRING_OPTION_DEFINITION) +#undef STRING_OPTION_DEFINITION + +#define BOOL_OPTION_DEFINITION(flag, variable) \ + static bool variable = false; \ + DEFINE_BOOL_OPTION(flag, variable) +BOOL_OPTIONS_LIST(BOOL_OPTION_DEFINITION) +#undef BOOL_OPTION_DEFINITION + +// clang-format off +static void PrintUsage() { + Syslog::PrintErr( +"Usage: analyze_snapshot [] [] \n" +" \n" +"Common options: \n" +"--help \n" +" Display this message. \n" +"--version \n" +" Print the SDK version. \n" +"--out \n" +" Path to generate the analysis results JSON. \n" +" \n" +"If omitting [] the VM parsing the snapshot is created with the \n" +"following default flags: \n" +"--enable_mirrors=false \n" +"--background_compilation \n" +"--lazy_async_stacks \n" +"--precompilation \n" +" \n" +"\n"); +} +// clang-format on + +const uint8_t* vm_snapshot_data = nullptr; +const uint8_t* vm_snapshot_instructions = nullptr; +const uint8_t* vm_isolate_data = nullptr; +const uint8_t* vm_isolate_instructions = nullptr; + +// Parse out the command line arguments. Returns -1 if the arguments +// are incorrect, 0 otherwise. +static int ParseArguments(int argc, + char** argv, + CommandLineOptions* vm_options, + CommandLineOptions* inputs) { + // Skip the binary name. + int i = 1; + + // Parse out the vm options. + while ((i < argc) && OptionProcessor::IsValidShortFlag(argv[i])) { + if (OptionProcessor::TryProcess(argv[i], vm_options)) { + i += 1; + continue; + } + vm_options->AddArgument(argv[i]); + i += 1; + } + + // Parse out the kernel inputs. + while (i < argc) { + inputs->AddArgument(argv[i]); + i++; + } + + if (help) { + PrintUsage(); + Platform::Exit(0); + } else if (version) { + Syslog::PrintErr("Dart SDK version: %s\n", Dart_VersionString()); + Platform::Exit(0); + } + + // Verify consistency of arguments. + if (inputs->count() < 1) { + Syslog::PrintErr("At least one input is required\n"); + return -1; + } + + if (out_path == nullptr) { + Syslog::PrintErr( + "Please specify an output path for analysis with the --out flag.\n\n"); + return -1; + } + + return 0; +} + +PRINTF_ATTRIBUTE(1, 2) static void PrintErrAndExit(const char* format, ...) { + va_list args; + va_start(args, format); + Syslog::VPrintErr(format, args); + va_end(args); + + Dart_ExitScope(); + Dart_ShutdownIsolate(); + exit(kErrorExitCode); +} + +static File* OpenFile(const char* filename) { + File* file = File::Open(nullptr, filename, File::kWriteTruncate); + if (file == nullptr) { + PrintErrAndExit("Error: Unable to write file: %s\n\n", filename); + } + return file; +} + +static void WriteFile(const char* filename, + const char* buffer, + const intptr_t size) { + File* file = OpenFile(filename); + RefCntReleaseScope rs(file); + if (!file->WriteFully(buffer, size)) { + PrintErrAndExit("Error: Unable to write file: %s\n\n", filename); + } +} + +int RunAnalyzer(int argc, char** argv) { + // Constant mirrors gen_snapshot binary, subject to change. + const int EXTRA_VM_ARGUMENTS = 7; + CommandLineOptions vm_options(argc + EXTRA_VM_ARGUMENTS); + CommandLineOptions inputs(argc); + // Parse command line arguments. + if (ParseArguments(argc, argv, &vm_options, &inputs) < 0) { + PrintUsage(); + return kErrorExitCode; + } + + // Initialize VM with default flags if none are provided. + // TODO(#47924): Implement auto-parsing of flags from the snapshot file. + if (vm_options.count() == 0) { + vm_options.AddArgument("--enable_mirrors=false"); + vm_options.AddArgument("--background_compilation"); + vm_options.AddArgument("--lazy_async_stacks"); + vm_options.AddArgument("--precompilation"); + } + + char* error = Dart_SetVMFlags(vm_options.count(), vm_options.arguments()); + if (error != nullptr) { + Syslog::PrintErr("Setting VM flags failed: %s\n", error); + free(error); + return kErrorExitCode; + } + + const char* script_name = nullptr; + script_name = inputs.GetArgument(0); + + // Dart_LoadELF will crash on nonexistant file non-gracefully + // even though it should return `nullptr`. + File* const file = File::Open(/*namespc=*/nullptr, script_name, File::kRead); + if (file == nullptr) { + Syslog::PrintErr("Snapshot file does not exist\n"); + return kErrorExitCode; + } + file->Release(); + + const char* loader_error = nullptr; + Dart_LoadedElf* loaded_elf = Dart_LoadELF( + script_name, 0, &loader_error, &vm_snapshot_data, + &vm_snapshot_instructions, &vm_isolate_data, &vm_isolate_instructions); + + if (loaded_elf == nullptr) { + Syslog::PrintErr("Failure calling Dart_LoadELF:\n%s\n", loader_error); + return kErrorExitCode; + } + + // Begin initialization + Dart_InitializeParams init_params = {}; + memset(&init_params, 0, sizeof(init_params)); + init_params.version = DART_INITIALIZE_PARAMS_CURRENT_VERSION; + init_params.vm_snapshot_data = vm_snapshot_data; + init_params.vm_snapshot_instructions = vm_snapshot_instructions; + + init_params.file_open = DartUtils::OpenFile; + init_params.file_read = DartUtils::ReadFile; + init_params.file_write = DartUtils::WriteFile; + init_params.file_close = DartUtils::CloseFile; + init_params.entropy_source = DartUtils::EntropySource; + + error = Dart_Initialize(&init_params); + if (error != nullptr) { + Syslog::PrintErr("VM initialization failed: %s\n", error); + free(error); + return kErrorExitCode; + } + + auto isolate_group_data = std::unique_ptr( + new IsolateGroupData(nullptr, nullptr, nullptr, false)); + + Dart_IsolateFlags isolate_flags; + Dart_IsolateFlagsInitialize(&isolate_flags); + // Null safety can be determined from the snapshot itself + isolate_flags.null_safety = + Dart_DetectNullSafety(nullptr, nullptr, nullptr, vm_snapshot_data, + vm_snapshot_instructions, nullptr, -1); + + Dart_CreateIsolateGroup(nullptr, nullptr, vm_isolate_data, + vm_isolate_instructions, &isolate_flags, + isolate_group_data.get(), + /*isolate_data=*/nullptr, &error); + + if (error != nullptr) { + Syslog::PrintErr("Dart_CreateIsolateGroup Error: %s\n", error); + free(error); + return kErrorExitCode; + } + + dart::snapshot_analyzer::Dart_SnapshotAnalyzerInformation info = { + vm_snapshot_data, vm_snapshot_instructions, vm_isolate_data, + vm_isolate_instructions}; + + char* out = NULL; + intptr_t out_len = 0; + + Dart_EnterScope(); + Dart_DumpSnapshotInformationAsJson(&out, &out_len, &info); + WriteFile(out_path, out, out_len); + // Since ownership of the JSON buffer is ours, free before we exit. + free(out); + Dart_ExitScope(); + Dart_ShutdownIsolate(); + // Unload our DartELF to avoid leaks + Dart_UnloadELF(loaded_elf); + return 0; +} +#endif +} // namespace bin +} // namespace dart + +int main(int argc, char** argv) { +#if !defined(DART_HOST_OS_FUCHSIA) + return dart::bin::RunAnalyzer(argc, argv); +#endif + dart::Syslog::PrintErr("Cannot run on Fuchsia.\n"); + return dart::bin::kErrorExitCode; +} diff --git a/runtime/include/analyze_snapshot_api.h b/runtime/include/analyze_snapshot_api.h new file mode 100644 index 00000000000..e02f461cc27 --- /dev/null +++ b/runtime/include/analyze_snapshot_api.h @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file + * for details. All rights reserved. Use of this source code is governed by a + * BSD-style license that can be found in the LICENSE file. + */ + +#ifndef RUNTIME_INCLUDE_ANALYZE_SNAPSHOT_API_H_ +#define RUNTIME_INCLUDE_ANALYZE_SNAPSHOT_API_H_ + +#include + +namespace dart { +namespace snapshot_analyzer { +typedef struct { + const uint8_t* vm_snapshot_data; + const uint8_t* vm_snapshot_instructions; + const uint8_t* vm_isolate_data; + const uint8_t* vm_isolate_instructions; +} Dart_SnapshotAnalyzerInformation; + +void Dart_DumpSnapshotInformationAsJson(char** buffer, + intptr_t* buffer_length, + Dart_SnapshotAnalyzerInformation* info); +} // namespace snapshot_analyzer +} // namespace dart + +#endif // RUNTIME_INCLUDE_ANALYZE_SNAPSHOT_API_H_ diff --git a/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart b/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart new file mode 100644 index 00000000000..6b735fd9843 --- /dev/null +++ b/runtime/tests/vm/dart/analyze_snapshot_binary_test.dart @@ -0,0 +1,214 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:async'; + +import 'package:expect/expect.dart'; +import 'package:native_stack_traces/elf.dart'; +import 'package:path/path.dart' as path; + +import 'use_flag_test_helper.dart'; + +// Used to ensure we don't have multiple equivalent calls to test. +final _seenDescriptions = {}; + +Future testAOT(String dillPath, + {bool useAsm = false, + bool forceDrops = false, + bool stripUtil = false, // Note: forced true if useAsm. + bool stripFlag = false, + bool disassemble = false}) async { + if (const bool.fromEnvironment('dart.vm.product') && disassemble) { + Expect.isFalse(disassemble, 'no use of disassembler in PRODUCT mode'); + } + + final analyzeSnapshot = path.join( + buildDir, + bool.fromEnvironment('dart.vm.product') + ? 'analyze_snapshot_product' + : 'analyze_snapshot'); + + // For assembly, we can't test the sizes of the snapshot sections, since we + // don't have a Mach-O reader for Mac snapshots and for ELF, the assembler + // merges the text/data sections and the VM/isolate section symbols may not + // have length information. Thus, we force external stripping so we can test + // the approximate size of the stripped snapshot. + if (useAsm) { + stripUtil = true; + } + + final descriptionBuilder = StringBuffer()..write(useAsm ? 'assembly' : 'elf'); + if (forceDrops) { + descriptionBuilder.write('-dropped'); + } + if (stripFlag) { + descriptionBuilder.write('-intstrip'); + } + if (stripUtil) { + descriptionBuilder.write('-extstrip'); + } + if (disassemble) { + descriptionBuilder.write('-disassembled'); + } + + final description = descriptionBuilder.toString(); + Expect.isTrue(_seenDescriptions.add(description), + "test configuration $description would be run multiple times"); + + await withTempDir('analyze_snapshot_binary-$description', + (String tempDir) async { + // Generate the snapshot + final snapshotPath = path.join(tempDir, 'test.snap'); + final commonSnapshotArgs = [ + if (stripFlag) '--strip', // gen_snapshot specific and not a VM flag. + if (forceDrops) ...[ + '--dwarf-stack-traces', + '--no-retain-function-objects', + '--no-retain-code-objects' + ], + if (disassemble) '--disassemble', // Not defined in PRODUCT mode. + dillPath, + ]; + + if (useAsm) { + final assemblyPath = path.join(tempDir, 'test.S'); + + await run(genSnapshot, [ + '--snapshot-kind=app-aot-assembly', + '--assembly=$assemblyPath', + ...commonSnapshotArgs, + ]); + + await assembleSnapshot(assemblyPath, snapshotPath); + } else { + await run(genSnapshot, [ + '--snapshot-kind=app-aot-elf', + '--elf=$snapshotPath', + ...commonSnapshotArgs, + ]); + } + + print("Snapshot generated at $snapshotPath."); + + // May not be ELF, but another format. + final elf = Elf.fromFile(snapshotPath); + if (!useAsm) { + Expect.isNotNull(elf); + } + + if (elf != null) { + // Verify some ELF file format parameters. + final textSections = elf.namedSections(".text"); + Expect.isNotEmpty(textSections); + Expect.isTrue( + textSections.length <= 2, "More text sections than expected"); + final dataSections = elf.namedSections(".rodata"); + Expect.isNotEmpty(dataSections); + Expect.isTrue( + dataSections.length <= 2, "More data sections than expected"); + } + + final analyzerOutputPath = path.join(tempDir, 'analyze_test.json'); + + // This will throw if exit code is not 0. + await run(analyzeSnapshot, [ + '--out=$analyzerOutputPath', + '$snapshotPath', + ]); + + final analyzerJsonBytes = await readFile(analyzerOutputPath); + final analyzerJson = json.decode(analyzerJsonBytes); + Expect.isFalse(analyzerJson.isEmpty); + Expect.isTrue(analyzerJson.keys + .toSet() + .containsAll(['snapshot_data', 'class_table', 'object_pool'])); + }); +} + +main() async { + void printSkip(String description) => + print('Skipping $description for ${path.basename(buildDir)} ' + 'on ${Platform.operatingSystem}' + + (clangBuildToolsDir == null ? ' without //buildtools' : '')); + + // We don't have access to the SDK on Android. + if (Platform.isAndroid) { + printSkip('all tests'); + return; + } + + await withTempDir('analyze_snapshot_binary', (String tempDir) async { + // We only need to generate the dill file once for all JIT tests. + final _thisTestPath = path.join(sdkDir, 'runtime', 'tests', 'vm', 'dart', + 'analyze_snapshot_binary_test.dart'); + + // We only need to generate the dill file once for all AOT tests. + final aotDillPath = path.join(tempDir, 'aot_test.dill'); + await run(genKernel, [ + '--aot', + '--platform', + platformDill, + ...Platform.executableArguments.where((arg) => + arg.startsWith('--enable-experiment=') || + arg == '--sound-null-safety' || + arg == '--no-sound-null-safety'), + '-o', + aotDillPath, + _thisTestPath + ]); + + // Just as a reminder for AOT tests: + // * If useAsm is true, then stripUtil is forced (as the assembler may add + // extra information that needs stripping), so no need to specify + // stripUtil for useAsm tests. + + // Test unstripped ELF generation directly. + await testAOT(aotDillPath); + await testAOT(aotDillPath, forceDrops: true); + + // Test flag-stripped ELF generation. + await testAOT(aotDillPath, stripFlag: true); + + // Since we can't force disassembler support after the fact when running + // in PRODUCT mode, skip any --disassemble tests. Do these tests last as + // they have lots of output and so the log will be truncated. + if (!const bool.fromEnvironment('dart.vm.product')) { + // Regression test for dartbug.com/41149. + await testAOT(aotDillPath, disassemble: true); + } + + // We neither generate assembly nor have a stripping utility on Windows. + if (Platform.isWindows) { + printSkip('external stripping and assembly tests'); + return; + } + + // The native strip utility on Mac OS X doesn't recognize ELF files. + if (Platform.isMacOS && clangBuildToolsDir == null) { + printSkip('ELF external stripping test'); + } else { + // Test unstripped ELF generation that is then externally stripped. + await testAOT(aotDillPath, stripUtil: true); + } + + // TODO(sstrickl): Currently we can't assemble for SIMARM64 on MacOSX. + // For example, the test runner still uses blobs for + // dartkp-mac-*-simarm64. Change assembleSnapshot and remove this check + // when we can. + if (Platform.isMacOS && buildDir.endsWith('SIMARM64')) { + printSkip('assembly tests'); + return; + } + // Test unstripped assembly generation that is then externally stripped. + await testAOT(aotDillPath, useAsm: true); + // Test stripped assembly generation that is then externally stripped. + await testAOT(aotDillPath, useAsm: true, stripFlag: true); + }); +} + +Future readFile(String file) { + return new File(file).readAsString(); +} diff --git a/runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart b/runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart new file mode 100644 index 00000000000..3299251b366 --- /dev/null +++ b/runtime/tests/vm/dart_2/analyze_snapshot_binary_test.dart @@ -0,0 +1,212 @@ +// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +// @dart = 2.9 + +import 'dart:convert'; +import 'dart:io'; +import 'dart:async'; + +import 'package:expect/expect.dart'; +import 'package:native_stack_traces/elf.dart'; +import 'package:path/path.dart' as path; + +import 'use_flag_test_helper.dart'; + +// Used to ensure we don't have multiple equivalent calls to test. +final _seenDescriptions = {}; + +Future testAOT(String dillPath, + {bool useAsm = false, + bool forceDrops = false, + bool stripUtil = false, // Note: forced true if useAsm. + bool stripFlag = false, + bool disassemble = false}) async { + if (const bool.fromEnvironment('dart.vm.product') && disassemble) { + Expect.isFalse(disassemble, 'no use of disassembler in PRODUCT mode'); + } + + final analyzeSnapshot = path.join( + buildDir, + bool.fromEnvironment('dart.vm.product') + ? 'analyze_snapshot_product' + : 'analyze_snapshot'); + + // For assembly, we can't test the sizes of the snapshot sections, since we + // don't have a Mach-O reader for Mac snapshots and for ELF, the assembler + // merges the text/data sections and the VM/isolate section symbols may not + // have length information. Thus, we force external stripping so we can test + // the approximate size of the stripped snapshot. + if (useAsm) { + stripUtil = true; + } + + final descriptionBuilder = StringBuffer()..write(useAsm ? 'assembly' : 'elf'); + if (forceDrops) { + descriptionBuilder.write('-dropped'); + } + if (stripFlag) { + descriptionBuilder.write('-intstrip'); + } + if (stripUtil) { + descriptionBuilder.write('-extstrip'); + } + if (disassemble) { + descriptionBuilder.write('-disassembled'); + } + + final description = descriptionBuilder.toString(); + Expect.isTrue(_seenDescriptions.add(description), + "test configuration $description would be run multiple times"); + + await withTempDir('analyze_snapshot_binary-$description', + (String tempDir) async { + // Generate the snapshot + final snapshotPath = path.join(tempDir, 'test.snap'); + final commonSnapshotArgs = [ + if (stripFlag) '--strip', // gen_snapshot specific and not a VM flag. + if (forceDrops) ...[ + '--dwarf-stack-traces', + '--no-retain-function-objects', + '--no-retain-code-objects' + ], + if (disassemble) '--disassemble', // Not defined in PRODUCT mode. + dillPath, + ]; + + if (useAsm) { + final assemblyPath = path.join(tempDir, 'test.S'); + + await run(genSnapshot, [ + '--snapshot-kind=app-aot-assembly', + '--assembly=$assemblyPath', + ...commonSnapshotArgs, + ]); + + await assembleSnapshot(assemblyPath, snapshotPath); + } else { + await run(genSnapshot, [ + '--snapshot-kind=app-aot-elf', + '--elf=$snapshotPath', + ...commonSnapshotArgs, + ]); + } + + print("Snapshot generated at $snapshotPath."); + + // May not be ELF, but another format. + final elf = Elf.fromFile(snapshotPath); + if (!useAsm) { + Expect.isNotNull(elf); + } + + if (elf != null) { + // Verify some ELF file format parameters. + final textSections = elf.namedSections(".text"); + Expect.isNotEmpty(textSections); + Expect.isTrue( + textSections.length <= 2, "More text sections than expected"); + final dataSections = elf.namedSections(".rodata"); + Expect.isNotEmpty(dataSections); + Expect.isTrue( + dataSections.length <= 2, "More data sections than expected"); + } + + final analyzerOutputPath = path.join(tempDir, 'analyze_test.json'); + + // This will throw if exit code is not 0. + await run(analyzeSnapshot, [ + '--out=$analyzerOutputPath', + '$snapshotPath', + ]); + + final analyzerJsonBytes = await readFile(analyzerOutputPath); + final analyzerJson = json.decode(analyzerJsonBytes); + Expect.isFalse(analyzerJson.isEmpty); + Expect.isTrue(analyzerJson.keys + .toSet() + .containsAll(['snapshot_data', 'class_table', 'object_pool'])); + }); +} + +main() async { + void printSkip(String description) => + print('Skipping $description for ${path.basename(buildDir)} ' + 'on ${Platform.operatingSystem}' + + (clangBuildToolsDir == null ? ' without //buildtools' : '')); + + // We don't have access to the SDK on Android. + if (Platform.isAndroid) { + printSkip('all tests'); + return; + } + + await withTempDir('analyze_snapshot_binary', (String tempDir) async { + // We only need to generate the dill file once for all JIT tests. + final _thisTestPath = path.join(sdkDir, 'runtime', 'tests', 'vm', 'dart_2', + 'analyze_snapshot_binary_test.dart'); + + // We only need to generate the dill file once for all AOT tests. + final aotDillPath = path.join(tempDir, 'aot_test.dill'); + await run(genKernel, [ + '--aot', + '--platform', + platformDill, + '-o', + aotDillPath, + _thisTestPath + ]); + + // Just as a reminder for AOT tests: + // * If useAsm is true, then stripUtil is forced (as the assembler may add + // extra information that needs stripping), so no need to specify + // stripUtil for useAsm tests. + + // Test unstripped ELF generation directly. + await testAOT(aotDillPath); + await testAOT(aotDillPath, forceDrops: true); + + // Test flag-stripped ELF generation. + await testAOT(aotDillPath, stripFlag: true); + + // Since we can't force disassembler support after the fact when running + // in PRODUCT mode, skip any --disassemble tests. Do these tests last as + // they have lots of output and so the log will be truncated. + if (!const bool.fromEnvironment('dart.vm.product')) { + // Regression test for dartbug.com/41149. + await testAOT(aotDillPath, disassemble: true); + } + + // We neither generate assembly nor have a stripping utility on Windows. + if (Platform.isWindows) { + printSkip('external stripping and assembly tests'); + return; + } + + // The native strip utility on Mac OS X doesn't recognize ELF files. + if (Platform.isMacOS && clangBuildToolsDir == null) { + printSkip('ELF external stripping test'); + } else { + // Test unstripped ELF generation that is then externally stripped. + await testAOT(aotDillPath, stripUtil: true); + } + + // TODO(sstrickl): Currently we can't assemble for SIMARM64 on MacOSX. + // For example, the test runner still uses blobs for + // dartkp-mac-*-simarm64. Change assembleSnapshot and remove this check + // when we can. + if (Platform.isMacOS && buildDir.endsWith('SIMARM64')) { + printSkip('assembly tests'); + return; + } + // Test unstripped assembly generation that is then externally stripped. + await testAOT(aotDillPath, useAsm: true); + // Test stripped assembly generation that is then externally stripped. + await testAOT(aotDillPath, useAsm: true, stripFlag: true); + }); +} + +Future readFile(String file) { + return new File(file).readAsString(); +} diff --git a/runtime/tests/vm/vm.status b/runtime/tests/vm/vm.status index 5fe1fbf7f76..a9e0d131fa0 100644 --- a/runtime/tests/vm/vm.status +++ b/runtime/tests/vm/vm.status @@ -65,6 +65,7 @@ dart_2/minimal_kernel_test: SkipSlow # gen_kernel is too slow with optimization_ dart_2/null_safety_autodetection_in_kernel_compiler_test: SkipSlow # gen_kernel is too slow with optimization_counter_threshold [ $builder_tag == tsan ] +dart/analyze_snapshot_binary_test: SkipSlow dart/appjit_cha_deopt_test: SkipSlow dart/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. dart/regress_40462_test: SkipSlow @@ -72,6 +73,7 @@ dart/regress_40753_test: Skip # This test crashes on the bot, but not locally, a dart/trigger_gc_in_native_test: Skip # This test crashes on the bot, but not locally, and infrastructure repeatly fails to locate its coredump. dart/use_strip_flag_test: Pass, Slow # This test can take a longer time to complete. dart/v8_snapshot_profile_writer_test: SkipSlow +dart_2/analyze_snapshot_binary_test: SkipSlow dart_2/appjit_cha_deopt_test: SkipSlow dart_2/hash_map_probes_limit_test: SkipSlow # Test includes large program compilation. dart_2/regress_40462_test: SkipSlow @@ -325,6 +327,11 @@ cc/CodeExecutability: SkipByDesign # --dual-map-code not supported on non-Linux/ dart/v8_snapshot_profile_writer_test: SkipByDesign # Only relevant for AOT. Doesn't work in cross-compilation (has to run on the host). On Linux/simarm64 and Linux/simarm this test requires buildtools/clang which is not always available on testing shards. dart_2/v8_snapshot_profile_writer_test: SkipByDesign # Only relevant for AOT. Doesn't work in cross-compilation (has to run on the host). On Linux/simarm64 and Linux/simarm this test requires buildtools/clang which is not always available on testing shards. +# Currently this is only supported on 64-bit linux systems with precompilation +[ $arch == ia32 || $arch == simarm || $arch == simarm64 || $arch == simarm64c || $arch == simriscv32 || $arch == simriscv64 || $compiler != dartk || $system == fuchsia || $system != linux ] +dart/analyze_snapshot_binary_test: SkipByDesign # Only run on 64bit AOT on standard architectures +dart_2/analyze_snapshot_binary_test: SkipByDesign # Only run on 64bit AOT on standard architectures + # On the simluator stack traces produced by the Profiler do not match # up with the real Dart stack trace and hence we don't get correct # symbol names. diff --git a/runtime/vm/analyze_snapshot_api_impl.cc b/runtime/vm/analyze_snapshot_api_impl.cc new file mode 100644 index 00000000000..ef73e763405 --- /dev/null +++ b/runtime/vm/analyze_snapshot_api_impl.cc @@ -0,0 +1,202 @@ +// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +#include "include/analyze_snapshot_api.h" +#include "vm/dart_api_impl.h" +#include "vm/json_writer.h" +#include "vm/object.h" +#include "vm/object_store.h" +#include "vm/thread.h" + +namespace dart { +namespace snapshot_analyzer { +void DumpClassTable(Thread* thread, dart::JSONWriter* js) { + auto class_table = thread->isolate_group()->class_table(); + + Class& cls = Class::Handle(); + String& name = String::Handle(); + js->OpenArray("class_table"); + + for (intptr_t i = 1; i < class_table->NumCids(); i++) { + if (!class_table->HasValidClassAt(i)) { + continue; + } + cls = class_table->At(i); + if (!cls.IsNull()) { + name = cls.Name(); + js->OpenObject(); + js->PrintProperty("id", i); + js->PrintProperty("name", name.ToCString()); + + // Note: Some meta info is stripped from the snapshot, it's important + // to check every field for NULL to avoid segfaults. + const Library& library = Library::Handle(cls.library()); + if (!library.IsNull()) { + String& lib_name = String::Handle(); + lib_name = String::NewFormatted( + Heap::kOld, "%s%s", String::Handle(library.url()).ToCString(), + String::Handle(library.private_key()).ToCString()); + js->PrintProperty("library", lib_name.ToCString()); + } + + const AbstractType& super_type = AbstractType::Handle(cls.super_type()); + if (super_type.IsNull()) { + } else { + const String& super_name = String::Handle(super_type.Name()); + js->PrintProperty("super_class", super_name.ToCString()); + } + + const Array& interfaces_array = Array::Handle(cls.interfaces()); + if (!interfaces_array.IsNull()) { + if (interfaces_array.Length() > 0) { + js->OpenArray("interfaces"); + AbstractType& interface = AbstractType::Handle(); + intptr_t len = interfaces_array.Length(); + for (intptr_t i = 0; i < len; i++) { + interface ^= interfaces_array.At(i); + js->PrintValue(interface.ToCString()); + } + js->CloseArray(); + } + } + const Array& functions_array = Array::Handle(cls.functions()); + if (!functions_array.IsNull()) { + if (functions_array.Length() > 0) { + js->OpenArray("functions"); + Function& function = Function::Handle(); + intptr_t len = functions_array.Length(); + for (intptr_t i = 0; i < len; i++) { + function ^= functions_array.At(i); + if (function.IsNull() || !function.HasCode()) { + continue; + } + const Code& code = Code::Handle(function.CurrentCode()); + intptr_t size = code.Size(); + + // Note: Some entry points here will be pointing to the VM + // instructions buffer. + + // Note: code_entry will contain the address in the memory + // In order to resolve it to a relative offset in the instructions + // buffer we need to pick the base address and substract it from + // the entry point address. + auto code_entry = code.EntryPoint(); + // On different architectures the type of the underlying + // dart::uword can result in an unsigned long long vs unsigned long + // mismatch. + uint64_t code_addr = static_cast(code_entry); + js->OpenObject(); + js->PrintProperty("name", function.ToCString()); + js->PrintfProperty("code_entry", "0x%" PRIx64 "", code_addr); + js->PrintProperty("size", size); + js->CloseObject(); + } + js->CloseArray(); + } + } + const Array& fields_array = Array::Handle(cls.fields()); + if (fields_array.IsNull()) { + } else { + if (fields_array.Length() > 0) { + js->OpenArray("fields"); + Field& field = Field::Handle(); + for (intptr_t i = 0; i < fields_array.Length(); i++) { + field ^= fields_array.At(i); + js->PrintValue(field.ToCString()); + } + js->CloseArray(); + } + } + } + js->CloseObject(); + } + js->CloseArray(); +} +void DumpObjectPool(Thread* thread, dart::JSONWriter* js) { + js->OpenArray("object_pool"); + + auto pool_ptr = thread->isolate_group()->object_store()->global_object_pool(); + const auto& pool = ObjectPool::Handle(ObjectPool::RawCast(pool_ptr)); + for (intptr_t i = 0; i < pool.Length(); i++) { + auto type = pool.TypeAt(i); + // Only interested in tagged objects. + // All these checks are required otherwise ToCString() will segfault. + if (type != ObjectPool::EntryType::kTaggedObject) { + continue; + } + + auto entry = pool.ObjectAt(i); + if (!entry.IsHeapObject()) { + continue; + } + + intptr_t cid = entry.GetClassId(); + + switch (cid) { + case kOneByteStringCid: { + js->OpenObject(); + js->PrintProperty("type", "kOneByteString"); + js->PrintProperty("id", i); + js->PrintProperty("offset", pool.element_offset(i)); + js->PrintProperty("value", Object::Handle(entry).ToCString()); + js->CloseObject(); + break; + } + case kTwoByteStringCid: { + // TODO(#47924): Add support. + break; + } + default: + // TODO(#47924): Investigate other types of objects to parse. + break; + } + } + js->CloseArray(); +} +// TODO(#47924): Add processing of the entires in the dispatch table. +// Below is an example skeleton +// void DumpDispatchTable(dart::Thread* thread) { +// auto dispatch = thread->isolate_group()->dispatch_table(); +// auto length = dispatch->length(); +// We must unbias the array entries so we don't crash on null access. +// auto entries = dispatch->ArrayOrigin() - DispatchTable::OriginElement(); +// for (intptr_t i = 0; i < length; i++) { +// OS::Print("0x%lx at %ld\n", entries[i], i); +// } +// } + +void Dart_DumpSnapshotInformationAsJson( + char** buffer, + intptr_t* buffer_length, + Dart_SnapshotAnalyzerInformation* info) { + Thread* thread = Thread::Current(); + DARTSCOPE(thread); + JSONWriter js; + // Open empty object so output is valid/parsable JSON. + js.OpenObject(); + js.OpenObject("snapshot_data"); + // Base addreses of the snapshot data, useful to calculate relative offsets. + js.PrintfProperty("vm_data", "%p", info->vm_snapshot_data); + js.PrintfProperty("vm_instructions", "%p", info->vm_snapshot_instructions); + js.PrintfProperty("isolate_data", "%p", info->vm_isolate_data); + js.PrintfProperty("isolate_instructions", "%p", + info->vm_isolate_instructions); + js.CloseObject(); + + { + // Debug builds assert that our thread has a lock before accessing + // vm internal fields. + SafepointReadRwLocker ml(thread, thread->isolate_group()->program_lock()); + DumpClassTable(thread, &js); + DumpObjectPool(thread, &js); + } + + // Close our empty object. + js.CloseObject(); + + // Give ownership to caller. + js.Steal(buffer, buffer_length); +} +} // namespace snapshot_analyzer +} // namespace dart diff --git a/sdk/BUILD.gn b/sdk/BUILD.gn index 8e6a103c8dd..1241607920b 100644 --- a/sdk/BUILD.gn +++ b/sdk/BUILD.gn @@ -26,6 +26,7 @@ declare_args() { dart_stripped_binary = "dart" dart_precompiled_runtime_stripped_binary = "dart_precompiled_runtime_product" gen_snapshot_stripped_binary = "gen_snapshot_product" + analyze_snapshot_binary = "analyze_snapshot_product" } # The directory layout of the SDK is as follows: diff --git a/tools/gn.py b/tools/gn.py index 2b4371743aa..e54c331dec3 100755 --- a/tools/gn.py +++ b/tools/gn.py @@ -282,6 +282,8 @@ def ToGnArgs(args, mode, arch, target_os, sanitizer, verify_sdk_hash): 'exe.stripped/dart_precompiled_runtime_product') gn_args['gen_snapshot_stripped_binary'] = ( 'exe.stripped/gen_snapshot_product') + gn_args['analyze_snapshot_binary'] = ( + 'exe.stripped/analyze_snapshot_product') # Setup the user-defined sysroot. if UseSysroot(args, gn_args):