Use batch mode compilation for -cdartkp --strong

Preliminary testing shows, this increases performance by around 20%. The
main benefit is by re-using a warmed-up VM and not start one from
scratch for every compilation.

Going forward we can do more optimizations, e.g. reading the platform
dill file only once into memory (instead of repeatedly) ...

  => This requires us using the new state-full IKG compiler.

Change-Id: I74db7dbb1aa79289d1045ef41f960215cf5b3b35
Reviewed-on: https://dart-review.googlesource.com/28240
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>

pkg/vm/bin/gen_kernel.dart

@@ -2,12 +2,14 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:io';
 
 import 'package:args/args.dart' show ArgParser, ArgResults;
 import 'package:front_end/src/api_prototype/front_end.dart';
 import 'package:kernel/binary/ast_to_binary.dart';
 import 'package:kernel/kernel.dart' show Program;
+import 'package:kernel/src/tool/batch_util.dart' as batch_util;
 import 'package:kernel/target/targets.dart' show TargetFlags;
 import 'package:kernel/target/vm.dart' show VmTarget;
 import 'package:vm/kernel_front_end.dart' show compileToKernel;
@@ -43,12 +45,20 @@ const _severityCaptions = const <Severity, String>{
 };
 
 main(List<String> arguments) async {
+  if (arguments.isNotEmpty && arguments.last == '--batch') {
+    await runBatchModeCompiler();
+  } else {
+    exit(await compile(arguments));
+  }
+}
+
+Future<int> compile(List<String> arguments) async {
   final ArgResults options = _argParser.parse(arguments);
   final String platformKernel = options['platform'];
 
   if ((options.rest.length != 1) || (platformKernel == null)) {
     print(_usage);
-    exit(_badUsageExitCode);
+    return _badUsageExitCode;
   }
 
   final String filename = options.rest.single;
@@ -82,11 +92,41 @@ main(List<String> arguments) async {
       aot: aot);
 
   if ((errors > 0) || (program == null)) {
-    exit(_compileTimeErrorExitCode);
+    return _compileTimeErrorExitCode;
   }
 
   final IOSink sink = new File(kernelBinaryFilename).openWrite();
   final BinaryPrinter printer = new BinaryPrinter(sink);
   printer.writeProgramFile(program);
   await sink.close();
+
+  return 0;
+}
+
+Future runBatchModeCompiler() async {
+  await batch_util.runBatch((List<String> arguments) async {
+    // TODO(kustermann): Once we know where the new IKG api is and how to use
+    // it, we should take advantage of it.
+    //
+    // Important things to note:
+    //
+    //   * Our global transformations must never alter the AST structures which
+    //     the statefull IKG generator keeps across compilations.
+    //     => We need to make our own copy.
+    //
+    //   * We must ensure the stateful IKG generator keeps giving us all the
+    //     compile-time errors, warnings, hints for every compilation and we
+    //     report the compilation result accordingly.
+    //
+    final exitCode = await compile(arguments);
+    switch (exitCode) {
+      case 0:
+        return batch_util.CompilerOutcome.Ok;
+      case _compileTimeErrorExitCode:
+      case _badUsageExitCode:
+        return batch_util.CompilerOutcome.Fail;
+      default:
+        throw 'Could not obtain correct exit code from compiler.';
+    }
+  });
 }

pkg/vm/tool/gen_kernel

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+# Script for generating kernel files using Dart 2 pipeline: Fasta with
+# strong mode enabled.
+
+set -e
+
+function follow_links() {
+  file="$1"
+  while [ -h "$file" ]; do
+    # On Mac OS, readlink -f doesn't work.
+    file="$(readlink "$file")"
+  done
+  echo "$file"
+}
+
+# Unlike $0, $BASH_SOURCE points to the absolute path of this file.
+PROG_NAME="$(follow_links "$BASH_SOURCE")"
+
+# Handle the case where dart-sdk/bin has been symlinked to.
+CUR_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
+
+SDK_DIR="$CUR_DIR/../../.."
+
+if [[ `uname` == 'Darwin' ]]; then
+  DART="$SDK_DIR/tools/sdks/mac/dart-sdk/bin/dart"
+  OUT_DIR="$SDK_DIR/xcodebuild"
+else
+  DART="$SDK_DIR/tools/sdks/linux/dart-sdk/bin/dart"
+  OUT_DIR="$SDK_DIR/out"
+fi
+
+exec "$DART" "${SDK_DIR}/pkg/vm/bin/gen_kernel.dart" $@

tools/testing/dart/command.dart

@@ -52,14 +52,14 @@ class Command {
         workingDirectory: workingDirectory);
   }
 
-  static Command kernelCompilation(
+  static Command vmKernelCompilation(
       String outputFile,
       bool neverSkipCompilation,
       List<Uri> bootstrapDependencies,
       String executable,
       List<String> arguments,
       Map<String, String> environment) {
-    return new KernelCompilationCommand._(outputFile, neverSkipCompilation,
+    return new VMKernelCompilationCommand._(outputFile, neverSkipCompilation,
         bootstrapDependencies, executable, arguments, environment);
   }
 
@@ -282,15 +282,15 @@ class CompilationCommand extends ProcessCommand {
       deepJsonCompare(_bootstrapDependencies, other._bootstrapDependencies);
 }
 
-class KernelCompilationCommand extends CompilationCommand {
-  KernelCompilationCommand._(
+class VMKernelCompilationCommand extends CompilationCommand {
+  VMKernelCompilationCommand._(
       String outputFile,
       bool neverSkipCompilation,
       List<Uri> bootstrapDependencies,
       String executable,
       List<String> arguments,
       Map<String, String> environmentOverrides)
-      : super._('dartk', outputFile, neverSkipCompilation,
+      : super._('vm_compile_to_kernel', outputFile, neverSkipCompilation,
             bootstrapDependencies, executable, arguments, environmentOverrides);
 
   int get maxNumRetries => 1;

tools/testing/dart/command_output.dart

@@ -856,8 +856,8 @@ class DevCompilerCommandOutput extends CommandOutput {
   }
 }
 
-class KernelCompilationCommandOutput extends CompilationCommandOutput {
-  KernelCompilationCommandOutput(
+class VMKernelCompilationCommandOutput extends CompilationCommandOutput {
+  VMKernelCompilationCommandOutput(
       Command command,
       int exitCode,
       bool timedOut,
@@ -888,7 +888,7 @@ class KernelCompilationCommandOutput extends CompilationCommandOutput {
   }
 
   /// If the compiler was able to produce a Kernel IR file we want to run the
-  /// result on the Dart VM. We therefore mark the [KernelCompilationCommand]
+  /// result on the Dart VM. We therefore mark the [VMKernelCompilationCommand]
   /// as successful.
   ///
   /// This ensures we test that the DartVM produces correct CompileTime errors
@@ -954,8 +954,8 @@ CommandOutput createCommandOutput(Command command, int exitCode, bool timedOut,
   } else if (command is VmCommand) {
     return new VMCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, pid);
-  } else if (command is KernelCompilationCommand) {
-    return new KernelCompilationCommandOutput(
+  } else if (command is VMKernelCompilationCommand) {
+    return new VMKernelCompilationCommandOutput(
         command, exitCode, timedOut, stdout, stderr, time, compilationSkipped);
   } else if (command is AdbPrecompilationCommand) {
     return new VMCommandOutput(

tools/testing/dart/compiler_configuration.dart

@@ -602,20 +602,19 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration {
 
   Command computeCompileToKernelCommand(String tempDir, List<String> arguments,
       Map<String, String> environmentOverrides) {
-    var buildDir = _configuration.buildDirectory;
-    String exec = Platform.executable;
+    final genKernel =
+        Platform.script.resolve('../../../pkg/vm/tool/gen_kernel').toFilePath();
+    final dillFile = tempKernelFile(tempDir);
     var args = [
       '--packages=.packages',
-      'pkg/vm/bin/gen_kernel.dart',
-      '--platform=${buildDir}/vm_platform_strong.dill',
       '--aot',
+      '--platform=${_configuration.buildDirectory}/vm_platform_strong.dill',
       '-o',
-      tempKernelFile(tempDir),
+      dillFile,
     ];
-    args.addAll(arguments.where((name) => name.endsWith('.dart')));
-    return Command.compilation('compile_to_kernel', tempDir,
-        bootstrapDependencies(), exec, args, environmentOverrides,
-        alwaysCompile: !_useSdk);
+    args.add(arguments.where((name) => name.endsWith('.dart')).single);
+    return Command.vmKernelCompilation(dillFile, true, bootstrapDependencies(),
+        genKernel, args, environmentOverrides);
   }
 
   /// Creates a command to clean up large temporary kernel files.

tools/testing/dart/test_runner.dart

@@ -1145,10 +1145,10 @@ class CommandExecutorImpl implements CommandExecutor {
   Future<CommandOutput> _runCommand(Command command, int timeout) {
     if (command is BrowserTestCommand) {
       return _startBrowserControllerTest(command, timeout);
-    } else if (command is KernelCompilationCommand) {
-      // For now, we always run dartk in batch mode.
+    } else if (command is VMKernelCompilationCommand) {
+      // For now, we always run vm_compile_to_kernel in batch mode.
       var name = command.displayName;
-      assert(name == 'dartk');
+      assert(name == 'vm_compile_to_kernel');
       return _getBatchRunner(name)
           .runCommand(name, command, timeout, command.arguments);
     } else if (command is CompilationCommand &&
@@ -1331,7 +1331,7 @@ bool shouldRetryCommand(CommandOutput output) {
 
     // The dartk batch compiler sometimes runs out of memory. In such a case we
     // will retry running it.
-    if (command is KernelCompilationCommand) {
+    if (command is VMKernelCompilationCommand) {
       if (output.hasCrashed) {
         bool containsOutOfMemoryMessage(String line) {
           return line.contains('Exhausted heap space, trying to allocat');