[dart2wasm] Make dart compile wasm use AOT-compiled dart2wasm via subprocess

This reduces time for `dart compile wasm` on a hello world in

* `--no-optimize` mode from 8.2 to 1.8 seconds (0.6 sec via [0])
* `--optimize` mode from 9.2 to 3 seconds (1.6 sec via [0])

[0] pkg/dart2wasm/tool/compile_benchmark

Issue https://github.com/dart-lang/sdk/issues/54675

Change-Id: I47093e747f343b542bc7faa34e102c62657c7b81
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/347902
Reviewed-by: Slava Egorov <vegorov@google.com>
Commit-Queue: Martin Kustermann <kustermann@google.com>

pkg/dart2wasm/lib/dart2wasm.dart

@@ -45,6 +45,8 @@ final List<Option> options = [
   Flag("omit-type-checks",
       (o, value) => o.translatorOptions.omitTypeChecks = value,
       defaultsTo: _d.translatorOptions.omitTypeChecks),
+  Flag("verbose", (o, value) => o.translatorOptions.verbose = value,
+      defaultsTo: _d.translatorOptions.verbose),
   Flag("verify-type-checks",
       (o, value) => o.translatorOptions.verifyTypeChecks = value,
       defaultsTo: _d.translatorOptions.verifyTypeChecks),
@@ -143,5 +145,5 @@ WasmCompilerOptions parseArguments(List<String> arguments) {
 
 Future<int> main(List<String> args) async {
   WasmCompilerOptions options = parseArguments(args);
-  return generateWasm(options);
+  return generateWasm(options, errorPrinter: stderr.writeln);
 }

pkg/dart2wasm/lib/generate_wasm.dart

@@ -7,8 +7,8 @@ export 'package:dart2wasm/compiler_options.dart';
 typedef PrintError = void Function(String error);
 
 Future<int> generateWasm(WasmCompilerOptions options,
-    {bool verbose = false, PrintError errorPrinter = print}) async {
-  if (verbose) {
+    {PrintError errorPrinter = print}) async {
+  if (options.translatorOptions.verbose) {
     print('Running dart compile wasm...');
     print('  - input file name   = ${options.mainUri}');
     print('  - output file name  = ${options.outputFile}');

pkg/dart2wasm/lib/translator.dart

@@ -40,6 +40,7 @@ class TranslatorOptions {
   bool printWasm = false;
   bool minify = false;
   bool verifyTypeChecks = false;
+  bool verbose = false;
   int inliningLimit = 0;
   int? sharedMemoryMaxPages;
   List<int> watchPoints = [];

pkg/dartdev/lib/src/commands/compile.dart

@@ -7,7 +7,6 @@ import 'dart:io';
 
 import 'package:args/args.dart';
 import 'package:dart2native/generate.dart';
-import 'package:dart2wasm/generate_wasm.dart';
 import 'package:front_end/src/api_prototype/compiler_options.dart'
     show Verbosity;
 import 'package:path/path.dart' as path;
@@ -247,9 +246,7 @@ class CompileSnapshotCommand extends CompileSubcommandCommand {
 
     log.stdout('Compiling $sourcePath to $commandName file $outputFile.');
     // TODO(bkonyi): perform compilation in same process.
-    final process = await startDartProcess(sdk, buildArgs);
-    routeToStdout(process);
-    return process.exitCode;
+    return await runProcess([sdk.dart, ...buildArgs]);
   }
 }
 
@@ -413,9 +410,6 @@ class CompileWasmCommand extends CompileSubcommandCommand {
   static const String help =
       'Compile Dart to a WebAssembly/WasmGC module (EXPERIMENTAL).';
 
-  final String optimizer = path.join(
-      binDir.path, 'utils', Platform.isWindows ? 'wasm-opt.exe' : 'wasm-opt');
-
   CompileWasmCommand({bool verbose = false})
       : super(commandName, help, verbose, hidden: !verbose) {
     argParser
@@ -499,13 +493,14 @@ class CompileWasmCommand extends CompileSubcommandCommand {
     log.stdout(
         'The support may change, or be removed, with no advance notice.\n');
 
-    final libraries = path.absolute(sdk.sdkPath, 'lib', 'libraries.json');
-    if (!Sdk.checkArtifactExists(libraries)) {
-      return 255;
-    }
     final args = argResults!;
-    bool verbose = this.verbose || args['verbose'];
-    if (args['optimize'] && !Sdk.checkArtifactExists(optimizer)) {
+    final verbose = this.verbose || args['verbose'];
+    final optimize = args['optimize'];
+
+    if (!Sdk.checkArtifactExists(sdk.librariesJson) ||
+        !Sdk.checkArtifactExists(sdk.dartAotRuntime) ||
+        !Sdk.checkArtifactExists(sdk.dart2wasmSnapshot) ||
+        (optimize && !Sdk.checkArtifactExists(sdk.wasmOpt))) {
       return 255;
     }
 
@@ -536,37 +531,40 @@ class CompileWasmCommand extends CompileSubcommandCommand {
     final outputFileBasename =
         outputFile.substring(0, outputFile.length - '.wasm'.length);
 
-    final options = WasmCompilerOptions(
-      mainUri: Uri.file(path.absolute(sourcePath)),
-      outputFile: outputFile,
-    );
-    options.librariesSpecPath =
-        Uri.file(path.absolute(sdk.sdkPath, 'lib', 'libraries.json'));
-    options.sdkPath = Uri.file(path.absolute(sdk.sdkPath));
-    options.packagesPath = args[packagesOption.flag];
-    options.translatorOptions.enableAsserts = args['enable-asserts'];
-    options.translatorOptions.printWasm = args['print-wasm'];
-    options.translatorOptions.printKernel = args['print-kernel'];
-    options.translatorOptions.omitTypeChecks = args['omit-type-checks'];
-    options.translatorOptions.nameSection = args['name-section'];
+    final sdkPath = path.absolute(sdk.sdkPath);
+    final packages = args[packagesOption.flag];
+
+    int? maxPages;
     if (args['shared-memory'] != null) {
-      int? maxPages = int.tryParse(args['shared-memory']);
+      maxPages = int.tryParse(args['shared-memory']);
       if (maxPages == null) {
         usageException(
             'Error: The --shared-memory flag must specify a number!');
       }
-      options.translatorOptions.importSharedMemory = true;
-      options.translatorOptions.sharedMemoryMaxPages = maxPages;
     }
 
-    int result;
+    final dart2wasmCommand = [
+      sdk.dartAotRuntime,
+      sdk.dart2wasmSnapshot,
+      '--libraries-spec=${sdk.librariesJson}',
+      '--dart-sdk=$sdkPath',
+      if (verbose) '--verbose',
+      if (packages != null) '--packages=$packages',
+      if (args['enable-asserts']) '--enable-asserts',
+      if (args['print-wasm']) '--print-wasm',
+      if (args['print-kernel']) '--print-kernel',
+      if (args['omit-type-checks']) '--omit-type-checks',
+      if (args['name-section']) '--name-section',
+      if (maxPages != null) ...[
+        '--import-shared-memory',
+        '--shared-memory-max-pages=$maxPages',
+      ],
+      path.absolute(sourcePath),
+      outputFile,
+    ];
     try {
-      result = await generateWasm(
-        options,
-        verbose: verbose,
-        errorPrinter: (error) => log.stderr(error),
-      );
-      if (result != 0) return compileErrorExitCode;
+      final exitCode = await runProcess(dart2wasmCommand);
+      if (exitCode != 0) return compileErrorExitCode;
     } catch (e, st) {
       log.stderr('Error: Wasm compilation failed');
       log.stderr(e.toString());
@@ -586,10 +584,10 @@ class CompileWasmCommand extends CompileSubcommandCommand {
       ];
 
       if (verbose) {
-        log.stdout('Optimizing output with: $optimizer $flags');
+        log.stdout('Optimizing output with: ${sdk.wasmOpt} $flags');
       }
       final processResult = Process.runSync(
-        optimizer,
+        sdk.wasmOpt,
         [...flags, '-o', outputFile, unoptFile],
       );
       if (processResult.exitCode != 0) {
@@ -602,7 +600,7 @@ class CompileWasmCommand extends CompileSubcommandCommand {
     final mjsFile = '$outputFileBasename.mjs';
     log.stdout(
         "Generated wasm module '$outputFile', and JS init file '$mjsFile'.");
-    return result;
+    return 0;
   }
 }
 

pkg/dartdev/lib/src/commands/create.dart

@@ -113,19 +113,15 @@ class CreateCommand extends DartdevCommand {
 
     if (args['pub']) {
       log.stdout('');
-      var progress = log.progress('Running pub get');
-      var process = await startDartProcess(
-        sdk,
-        ['pub', 'get'],
-        cwd: dir,
-      );
+      final progress = log.progress('Running pub get');
 
       // Run 'pub get'. We display output from the pub command, but keep the
       // output terse. This is to give the user a sense of the work that pub
       // did without scrolling the previous stdout sections off the screen.
-      var buffer = StringBuffer();
-      routeToStdout(
-        process,
+      final buffer = StringBuffer();
+      final exitCode = await runProcess(
+        [sdk.dart, 'pub', 'get'],
+        cwd: dir,
         logToTrace: true,
         listener: (str) {
           // Filter lines like '+ multi_server_socket 1.0.2'.
@@ -134,8 +130,7 @@ class CreateCommand extends DartdevCommand {
           }
         },
       );
-      int code = await process.exitCode;
-      if (code != 0) return code;
+      if (exitCode != 0) return exitCode;
       progress.finish(showTiming: true);
       log.stdout(buffer.toString().trimRight());
     }

pkg/dartdev/lib/src/core.dart

@@ -101,41 +101,42 @@ Future<Process> startDartProcess(
   return Process.start(sdk.dart, arguments, workingDirectory: cwd);
 }
 
-void routeToStdout(
-  Process process, {
+Future<int> runProcess(
+  List<String> command, {
   bool logToTrace = false,
   void Function(String str)? listener,
-}) {
-  if (isDiagnostics) {
-    _streamLineTransform(process.stdout, (String line) {
-      logToTrace ? log.trace(line.trimRight()) : log.stdout(line.trimRight());
-      if (listener != null) listener(line);
-    });
-    _streamLineTransform(process.stderr, (String line) {
-      log.stderr(line.trimRight());
-      if (listener != null) listener(line);
-    });
-  } else {
-    _streamLineTransform(process.stdout, (String line) {
-      logToTrace ? log.trace(line.trimRight()) : log.stdout(line.trimRight());
-      if (listener != null) listener(line);
-    });
-
-    _streamLineTransform(process.stderr, (String line) {
-      log.stderr(line.trimRight());
+  String? cwd,
+}) async {
+  Future forward(Stream<List<int>> output, bool isStderr) {
+    return _streamLineTransform(output, (line) {
+      final trimmed = line.trimRight();
+      logToTrace
+          ? log.trace(trimmed)
+          : (isStderr ? log.stderr(trimmed) : log.stdout(trimmed));
       if (listener != null) listener(line);
     });
   }
+
+  log.trace(command.join(' '));
+  final process = await Process.start(command.first, command.skip(1).toList(),
+      workingDirectory: cwd);
+  final (_, _, exitCode) = await (
+    forward(process.stdout, false),
+    forward(process.stderr, true),
+    process.exitCode
+  ).wait;
+  return exitCode;
 }
 
-void _streamLineTransform(
+Future _streamLineTransform(
   Stream<List<int>> stream,
   Function(String line) handler,
 ) {
-  stream
+  return stream
       .transform(utf8.decoder)
       .transform(const LineSplitter())
-      .forEach(handler);
+      .listen(handler)
+      .asFuture();
 }
 
 /// A representation of a project on disk.

pkg/dartdev/lib/src/sdk.dart

@@ -49,6 +49,13 @@ class Sdk {
         'dart2js.dart.snapshot',
       );
 
+  String get dart2wasmSnapshot => path.absolute(
+        sdkPath,
+        'bin',
+        'snapshots',
+        'dart2wasm_product.snapshot',
+      );
+
   String get ddsSnapshot => path.absolute(
         sdkPath,
         'bin',
@@ -91,6 +98,11 @@ class Sdk {
         'devtools',
       );
 
+  String get wasmOpt => path.join(sdkPath, 'bin', 'utils',
+      Platform.isWindows ? 'wasm-opt.exe' : 'wasm-opt');
+
+  String get librariesJson => path.absolute(sdkPath, 'lib', 'libraries.json');
+
   static bool checkArtifactExists(String path, {bool logError = true}) {
     if (!File(path).existsSync()) {
       if (logError) {

pkg/dartdev/pubspec.yaml

@@ -15,7 +15,6 @@ dependencies:
   cli_util: any
   collection: any
   dart2native: any
-  dart2wasm: any
   dart_style: any
   dartdoc: any
   dds: any

pkg/test_runner/lib/src/command_output.dart

@@ -1146,7 +1146,8 @@ class Dart2WasmCompilerCommandOutput extends CompilationCommandOutput
   @override
   void _parseErrors() {
     var errors = <StaticError>[];
-    parseErrors(decodeUtf8(stdout), errors);
+    // We expect errors to be printed to `stderr` for dart2wasm.
+    parseErrors(decodeUtf8(stderr), errors);
     errors.forEach(addError);
   }
 }