[gardening] Add coredump archiving support to iso-stress builder.

This is an attempt to enable archiving of coredumps on the
"iso-stress" builder, since we're often unable to reproduce crashes from
that builder.

Issue https://github.com/dart-lang/sdk/issues/46823

TEST=Adds test infra.

Change-Id: I9b7276198db9a6c98a74f55d466bf832b03e24f8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/214407
Reviewed-by: Alexander Thomas <athom@google.com>
Reviewed-by: Slava Egorov <vegorov@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>

runtime/tests/concurrency/run_stress_test_shards.dart

@@ -12,6 +12,8 @@ import 'package:path/path.dart' as path;
 
 import '../vm/dart/snapshot_test_helper.dart';
 
+int crashCounter = 0;
+
 void forwardStream(Stream<List<int>> input, IOSink output) {
   // Print the information line-by-line.
   input
@@ -22,14 +24,25 @@ void forwardStream(Stream<List<int>> input, IOSink output) {
   });
 }
 
-Future<bool> run(String executable, List<String> args) async {
+class PotentialCrash {
+  final String test;
+  final int pid;
+  final List<String> binaries;
+  PotentialCrash(this.test, this.pid, this.binaries);
+}
+
+Future<bool> run(
+    String executable, List<String> args, List<PotentialCrash> crashes) async {
   print('Running "$executable ${args.join(' ')}"');
   final Process process = await Process.start(executable, args);
   forwardStream(process.stdout, stdout);
   forwardStream(process.stderr, stderr);
   final int exitCode = await process.exitCode;
   if (exitCode != 0) {
+    final crashNr = crashCounter++;
     print('=> Running "$executable ${args.join(' ')}" failed with $exitCode');
+    print('=> Possible crash $crashNr (pid: ${process.pid})');
+    crashes.add(PotentialCrash('crash-$crashNr', process.pid, [executable]));
     io.exitCode = 255; // Make this shard fail.
     return false;
   }
@@ -37,7 +50,7 @@ Future<bool> run(String executable, List<String> args) async {
 }
 
 abstract class TestRunner {
-  Future runTest();
+  Future runTest(List<PotentialCrash> crashes);
 }
 
 class JitTestRunner extends TestRunner {
@@ -46,8 +59,8 @@ class JitTestRunner extends TestRunner {
 
   JitTestRunner(this.buildDir, this.arguments);
 
-  Future runTest() async {
-    await run('$buildDir/dart', arguments);
+  Future runTest(List<PotentialCrash> crashes) async {
+    await run('$buildDir/dart', arguments, crashes);
   }
 }
 
@@ -58,19 +71,62 @@ class AotTestRunner extends TestRunner {
 
   AotTestRunner(this.buildDir, this.arguments, this.aotArguments);
 
-  Future runTest() async {
+  Future runTest(List<PotentialCrash> crashes) async {
     await withTempDir((String dir) async {
       final elfFile = path.join(dir, 'app.elf');
 
-      if (await run('$buildDir/gen_snapshot',
-          ['--snapshot-kind=app-aot-elf', '--elf=$elfFile', ...arguments])) {
-        await run(
-            '$buildDir/dart_precompiled_runtime', [...aotArguments, elfFile]);
+      if (await run(
+          '$buildDir/gen_snapshot',
+          ['--snapshot-kind=app-aot-elf', '--elf=$elfFile', ...arguments],
+          crashes)) {
+        await run('$buildDir/dart_precompiled_runtime',
+            [...aotArguments, elfFile], crashes);
       }
     });
   }
 }
 
+// Produces a name that tools/utils.py:BaseCoredumpArchiver supports.
+String getArchiveName(String binary) {
+  final parts = binary.split(Platform.pathSeparator);
+  late String mode;
+  late String arch;
+  final buildDir = parts[1];
+  for (final prefix in ['Release', 'Debug', 'Product']) {
+    if (buildDir.startsWith(prefix)) {
+      mode = prefix.toLowerCase();
+      arch = buildDir.substring(prefix.length);
+    }
+  }
+  final name = parts.skip(2).join('__');
+  return 'binary.${mode}_${arch}_${name}';
+}
+
+void writeUnexpectedCrashesFile(List<PotentialCrash> crashes) {
+  // The format of this file is:
+  //
+  //     test-name,pid,binary-file1,binary-file2,...
+  //
+  const unexpectedCrashesFile = 'unexpected-crashes';
+
+  final buffer = StringBuffer();
+  final Set<String> archivedBinaries = {};
+  for (final crash in crashes) {
+    buffer.write('${crash.test},${crash.pid}');
+    for (final binary in crash.binaries) {
+      final archivedName = getArchiveName(binary);
+      buffer.write(',$archivedName');
+      if (!archivedBinaries.contains(archivedName)) {
+        File(binary).copySync(archivedName);
+        archivedBinaries.add(archivedName);
+      }
+    }
+    buffer.writeln();
+  }
+
+  File(unexpectedCrashesFile).writeAsStringSync(buffer.toString());
+}
+
 const int tsanShards = 200;
 
 final configurations = <TestRunner>[
@@ -120,11 +176,14 @@ main(List<String> arguments) async {
     ..addOption('shards', help: 'number of shards used', defaultsTo: '1')
     ..addOption('shard', help: 'shard id', defaultsTo: '1')
     ..addOption('output-directory',
-        help: 'unused parameter to make sharding infra work', defaultsTo: '');
+        help: 'unused parameter to make sharding infra work', defaultsTo: '')
+    ..addFlag('copy-coredumps',
+        help: 'whether to copy binaries for coredumps', defaultsTo: false);
 
   final options = parser.parse(arguments);
   final shards = int.parse(options['shards']);
   final shard = int.parse(options['shard']) - 1;
+  final copyCoredumps = options['copy-coredumps'] as bool;
 
   // Tasks will eventually be killed if they do not have any output for some
   // time. So we'll explicitly print something every 4 minutes.
@@ -140,8 +199,12 @@ main(List<String> arguments) async {
         thisShardsConfigurations.add(configurations[i]);
       }
     }
+    final crashes = <PotentialCrash>[];
     for (final config in thisShardsConfigurations) {
-      await config.runTest();
+      await config.runTest(crashes);
+    }
+    if (!crashes.isEmpty && copyCoredumps) {
+      writeUnexpectedCrashesFile(crashes);
     }
   } finally {
     timer.cancel();

tools/bots/test_matrix.json

@@ -3912,9 +3912,11 @@
         },
         {
           "name": "Run Isolate Stress Tests",
-          "script": "out/ReleaseX64/dart",
+          "script": "tools/run_with_coredumps_enabled.py",
           "arguments": [
-            "runtime/tests/concurrency/run_stress_test_shards.dart"
+            "out/ReleaseX64/dart",
+            "runtime/tests/concurrency/run_stress_test_shards.dart",
+            "--copy-coredumps"
           ],
           "shards": 10,
           "fileset": "vm-kernel"

tools/run_with_coredumps_enabled.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+from contextlib import ExitStack
+import subprocess
+import sys
+
+import utils
+
+
+def Main():
+    args = sys.argv[1:]
+
+    with ExitStack() as stack:
+        for ctx in utils.CoreDumpArchiver(args):
+            stack.enter_context(ctx)
+        exit_code = subprocess.call(args)
+
+    utils.DiagnoseExitCode(exit_code, args)
+    return exit_code
+
+
+if __name__ == '__main__':
+    sys.exit(Main())