[benchmark] Add support for benchmarking DDC.

This change adds the required support to benchmark DDC just like dart2js.

The ddb tool gains the following options:

  --compile-vm-options=VM-OPTIONS
      The value of the DART_VM_OPTIONS environment variable set when running
      dartdevc. This option is required to measure the memory use of
      dartdevc while compiling.

  --packages=PACKAGES-FILE
      The .packages file to use when compiling. This option is required for
      special benchmarks with a .packages in a subdirectory and is generally
      useful.

  --out=OUTPUT-FILE
      Write the compiled javascript to this path and the other generated
      files along it. This option makes the benchmark configuration easier
      and is generally useful.

The dartdevc and dartdevc.bat scripts in the released Dart SDK now gains
support for the DART_VM_OPTIONS environment variable, just like the dart2js
scripts. This feature is needed to measure the memory usage of the Dart VM
executing dartdevc.

tools/bots/try_benchmarks.sh now tests that ddb can compile and run
benchmarks using the needed features.

Change-Id: Ib1ef07b0888a8c0cf900fe2fbb5697aab733db08
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/118440
Reviewed-by: Nate Bosch <nbosch@google.com>
Commit-Queue: Jonas Termansen <sortie@google.com>

pkg/dev_compiler/tool/ddb

@@ -62,7 +62,13 @@ void main(List<String> args) async {
     ..addOption('mode',
         help: 'Option to (compile|run|all).  Default is all (compile and run).',
         allowed: ['compile', 'run', 'all'],
-        defaultsTo: 'all');
+        defaultsTo: 'all')
+    ..addOption('compile-vm-options',
+        help: 'DART_VM_OPTIONS for the compilation VM.')
+    ..addOption('packages',
+        help: 'Where to find a package spec file.')
+    ..addOption('out',
+        help: 'Output file.');
 
   var options = parser.parse(args);
   if (options['help'] as bool) {
@@ -90,6 +96,7 @@ void main(List<String> args) async {
   var run = mode == 'run' || mode == 'all';
 
   var entry = p.canonicalize(options.rest.first);
+  var out = (options['out'] as String) ?? p.setExtension(entry, '.js');
   var libRoot = p.dirname(entry);
   var basename = p.basenameWithoutExtension(entry);
   var libname = kernel
@@ -135,8 +142,12 @@ void main(List<String> args) async {
       // Use built snapshot.
       command = p.join(dartSdk, 'bin', command);
     }
-    var process =
-        await Process.start(command, args, mode: ProcessStartMode.inheritStdio);
+    var process = await Process.start(command, args,
+        mode: ProcessStartMode.inheritStdio,
+        environment: <String, String>{
+          if (options['compile-vm-options'] != null)
+            'DART_VM_OPTIONS': options['compile-vm-options']
+        });
     if (await process.exitCode != 0) exit(await process.exitCode);
   }
 
@@ -185,8 +196,9 @@ void main(List<String> args) async {
         '--library-root=$libRoot',
       for (var summary in summaries) '--summary=$summary',
       for (var experiment in experiments) '--enable-experiment=$experiment',
+      if (options['packages'] != null) '--packages=${options['packages']}',
       '-o',
-      p.setExtension(entry, '.js'),
+      out,
       entry
     ];
 
@@ -226,7 +238,7 @@ void main(List<String> args) async {
   });
 </script>
 ''';
-      var htmlFile = p.setExtension(entry, '.html');
+      var htmlFile = p.setExtension(out, '.html');
       File(htmlFile).writeAsStringSync(html);
       var tmp = p.join(Directory.systemTemp.path, 'ddc');
 
@@ -262,7 +274,7 @@ try {
   process.exit(1);
 }
 ''';
-      var nodeFile = p.setExtension(entry, '.run.js');
+      var nodeFile = p.setExtension(out, '.run.js');
       File(nodeFile).writeAsStringSync(runjs);
       var nodeBinary = binary ?? 'node';
       var process = await Process.start(
@@ -274,7 +286,7 @@ try {
       // Fix SDK import.  `d8` doesn't let us set paths, so we need a full path
       // to the SDK.
 
-      var jsFile = File(p.setExtension(entry, '.js'));
+      var jsFile = File(out);
       var jsContents = jsFile.readAsStringSync();
       jsContents = jsContents.replaceFirst(
           "from 'dart_sdk.js'", "from '$sdkJsPath/dart_sdk.js'");
@@ -291,7 +303,7 @@ try {
   console.error(e);
 }
 ''';
-      var d8File = p.setExtension(entry, '.d8.js');
+      var d8File = p.setExtension(out, '.d8.js');
       File(d8File).writeAsStringSync(runjs);
       var d8Binary = binary ?? p.join(dartCheckoutPath, _d8executable);
       var process = await Process.start(d8Binary, ['--module', d8File],

sdk/bin/dartdevc_sdk

@@ -25,4 +25,14 @@ SNAPSHOT="$BIN_DIR/snapshots/dartdevc.dart.snapshot"
 
 # We are running the snapshot in the built SDK.
 DART="$BIN_DIR/dart"
-exec "$DART" "$SNAPSHOT" "$@"
+
+unset EXTRA_VM_OPTIONS
+declare -a EXTRA_VM_OPTIONS
+
+# We allow extra vm options to be passed in through an environment variable.
+if [[ $DART_VM_OPTIONS ]]; then
+  read -a OPTIONS <<< "$DART_VM_OPTIONS"
+  EXTRA_VM_OPTIONS+=("${OPTIONS[@]}")
+fi
+
+exec "$DART" "${EXTRA_VM_OPTIONS[@]}" "$SNAPSHOT" "$@"

sdk/bin/dartdevc_sdk.bat

@@ -22,7 +22,14 @@ if %SDK_DIR:~-1%==\ set SDK_DIR=%SDK_DIR:~0,-1%
 
 set SDK_ARG=--dart-sdk=%SDK_DIR%
 
-"%DART%" "%SNAPSHOT%" "%SDK_ARG%" %*
+set EXTRA_VM_OPTIONS=
+
+rem We allow extra vm options to be passed in through an environment variable.
+if not "_%DART_VM_OPTIONS%_" == "__" (
+  set EXTRA_VM_OPTIONS=%EXTRA_VM_OPTIONS% %DART_VM_OPTIONS%
+)
+
+"%DART%" %EXTRA_VM_OPTIONS% "%SNAPSHOT%" "%SDK_ARG%" %*
 
 endlocal
 

sdk_nnbd/bin/dartdevc_sdk

@@ -25,4 +25,14 @@ SNAPSHOT="$BIN_DIR/snapshots/dartdevc.dart.snapshot"
 
 # We are running the snapshot in the built SDK.
 DART="$BIN_DIR/dart"
-exec "$DART" "$SNAPSHOT" "$@"
+
+unset EXTRA_VM_OPTIONS
+declare -a EXTRA_VM_OPTIONS
+
+# We allow extra vm options to be passed in through an environment variable.
+if [[ $DART_VM_OPTIONS ]]; then
+  read -a OPTIONS <<< "$DART_VM_OPTIONS"
+  EXTRA_VM_OPTIONS+=("${OPTIONS[@]}")
+fi
+
+exec "$DART" "${EXTRA_VM_OPTIONS[@]}" "$SNAPSHOT" "$@"

sdk_nnbd/bin/dartdevc_sdk.bat

@@ -22,7 +22,14 @@ if %SDK_DIR:~-1%==\ set SDK_DIR=%SDK_DIR:~0,-1%
 
 set SDK_ARG=--dart-sdk=%SDK_DIR%
 
-"%DART%" "%SNAPSHOT%" "%SDK_ARG%" %*
+set EXTRA_VM_OPTIONS=
+
+rem We allow extra vm options to be passed in through an environment variable.
+if not "_%DART_VM_OPTIONS%_" == "__" (
+  set EXTRA_VM_OPTIONS=%EXTRA_VM_OPTIONS% %DART_VM_OPTIONS%
+)
+
+"%DART%" %EXTRA_VM_OPTIONS% "%SNAPSHOT%" "%SDK_ARG%" %*
 
 endlocal
 

tools/bots/try_benchmarks.sh

@@ -216,6 +216,8 @@ EOF
     third_party/d8/linux/ia32/d8 --stack_size=1024 sdk/lib/_internal/js_runtime/lib/preambles/d8.js out.js
     out/ReleaseIA32/dart-sdk/bin/dart2js --use-kernel --packages=.packages --out=out.js -m hello.dart
     third_party/d8/linux/ia32/d8 --stack_size=1024 sdk/lib/_internal/js_runtime/lib/preambles/d8.js out.js
+    out/ReleaseIA32/dart-sdk/bin/dart pkg/dev_compiler/tool/ddb -r d8 -b third_party/d8/linux/ia32/d8 hello.dart
+    out/ReleaseIA32/dart-sdk/bin/dart pkg/dev_compiler/tool/ddb -r d8 -b third_party/d8/linux/ia32/d8 --kernel hello.dart
     out/ReleaseIA32/dart pkg/front_end/tool/perf.dart parse hello.dart
     out/ReleaseIA32/dart pkg/front_end/tool/perf.dart scan hello.dart
     out/ReleaseIA32/dart pkg/front_end/tool/fasta_perf.dart kernel_gen_e2e hello.dart
@@ -395,6 +397,8 @@ EOF
     out/ReleaseX64/dart --profile-period=10000 --packages=.packages --enable-interpreter --compilation-counter-threshold=-1 hello.dart
     out/ReleaseX64/dart --profile-period=10000 --packages=.packages --use-bytecode-compiler hello.dart
     out/ReleaseX64/dart --profile-period=10000 --packages=.packages --use-bytecode-compiler --optimization-counter-threshold=-1 hello.dart
+    out/ReleaseX64/dart-sdk/bin/dart pkg/dev_compiler/tool/ddb -r d8 -b third_party/d8/linux/ia32/d8 --mode=compile --compile-vm-options=--print-metrics --packages=.packages --out out.js hello.dart
+    out/ReleaseX64/dart-sdk/bin/dart pkg/dev_compiler/tool/ddb -r d8 -b third_party/d8/linux/ia32/d8 --mode=compile --compile-vm-options=--print-metrics --packages=.packages --out out.js --kernel hello.dart
     out/ReleaseX64/dart pkg/front_end/tool/perf.dart parse hello.dart
     out/ReleaseX64/dart pkg/front_end/tool/perf.dart scan hello.dart
     out/ReleaseX64/dart pkg/front_end/tool/fasta_perf.dart kernel_gen_e2e hello.dart