Add flags to configure incremental_perf

This lets us benchmark separately:
- IKG implementations (the default based on kernel-driver or the minimal implementation)
- strong and non-strong mode

This also adds commands to try_benchmarks that highlight how these options will be used.

Change-Id: I5ce2c4563b7e79c33d78df7fd87be76f5d47e3f4
Reviewed-on: https://dart-review.googlesource.com/21320
Reviewed-by: Jonas Termansen <sortie@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
Commit-Queue: Sigmund Cherem <sigmund@google.com>

pkg/front_end/tool/incremental_perf.dart

@@ -56,6 +56,7 @@ import 'package:front_end/physical_file_system.dart';
 import 'package:front_end/src/byte_store/protected_file_byte_store.dart';
 import 'package:kernel/target/flutter.dart';
 import 'package:kernel/target/targets.dart';
+import 'package:kernel/target/vm.dart';
 
 main(List<String> args) async {
   var options = argParser.parse(args);
@@ -71,10 +72,15 @@ main(List<String> args) async {
       parse(JSON.decode(new File.fromUri(editsUri).readAsStringSync()));
 
   var overlayFs = new OverlayFileSystem();
+  var targetFlags = new TargetFlags(strongMode: options['mode'] == 'strong');
   var compilerOptions = new CompilerOptions()
     ..fileSystem = overlayFs
+    ..strongMode = (options['mode'] == 'strong')
     ..reportMessages = true
-    ..onError = onErrorHandler;
+    ..onError = onErrorHandler
+    ..target = options['target'] == 'flutter'
+        ? new FlutterTarget(targetFlags)
+        : new VmTarget(targetFlags);
 
   if (options['sdk-summary'] != null) {
     compilerOptions.sdkSummary = _resolveOverlayUri(options["sdk-summary"]);
@@ -82,16 +88,14 @@ main(List<String> args) async {
     compilerOptions.librariesSpecificationUri =
         _resolveOverlayUri(options["sdk-library-specification"]);
   }
-  if (options['target'] == 'flutter') {
-    compilerOptions.target = new FlutterTarget(new TargetFlags());
-  }
 
   var dir = Directory.systemTemp.createTempSync('ikg-cache');
   compilerOptions.byteStore = createByteStore(options['cache'], dir.path);
 
   var timer1 = new Stopwatch()..start();
-  var generator =
-      await IncrementalKernelGenerator.newInstance(compilerOptions, entryUri);
+  var generator = await IncrementalKernelGenerator.newInstance(
+      compilerOptions, entryUri,
+      useMinimalGenerator: options['implementation'] == 'minimal');
 
   var delta = await generator.computeDelta();
   generator.acceptLastDelta();
@@ -254,6 +258,14 @@ ArgParser argParser = new ArgParser()
       help: 'caching policy used by the compiler',
       defaultsTo: 'protected',
       allowed: ['evicting', 'memory', 'protected'])
+  ..addOption('mode',
+      help: 'whether to run in strong or legacy mode',
+      defaultsTo: 'strong',
+      allowed: ['legacy', 'strong'])
+  ..addOption('implementation',
+      help: 'incremental compiler implementation to use',
+      defaultsTo: 'driver',
+      allowed: ['driver', 'minimal'])
   ..addOption('sdk-summary', help: 'Location of the sdk outline.dill file')
   ..addOption('sdk-library-specification',
       help: 'Location of the '

pkg/front_end/tool/incremental_perf_test.dart

@@ -4,7 +4,6 @@
 
 /// Test to ensure that incremental_perf.dart is running without errors.
 
-import 'dart:async';
 import 'dart:io';
 import 'package:front_end/src/compute_platform_binaries_location.dart'
     show computePlatformBinariesLocation;
@@ -16,12 +15,42 @@ main() async {
       "vm_platform.dill");
 
   final ikgBenchmarks = Platform.script.resolve('../benchmarks/ikg/');
-  await runExample(sdkOutline, ikgBenchmarks.resolve('hello.dart'),
-      ikgBenchmarks.resolve('hello.edits.json'));
-  await runExample(sdkOutline, ikgBenchmarks.resolve('dart2js.dart'),
-      ikgBenchmarks.resolve('dart2js.edits.json'));
-}
+  final helloEntry = ikgBenchmarks.resolve('hello.dart');
+  final helloEdits = ikgBenchmarks.resolve('hello.edits.json');
+  await m.main(['--sdk-summary', '$sdkOutline', '$helloEntry', '$helloEdits']);
+  await m.main([
+    '--sdk-summary',
+    '$sdkOutline',
+    '--mode=legacy',
+    '$helloEntry',
+    '$helloEdits'
+  ]);
+  await m.main([
+    '--sdk-summary',
+    '$sdkOutline',
+    '--implementation=minimal',
+    '$helloEntry',
+    '$helloEdits'
+  ]);
 
-Future runExample(Uri sdkOutline, Uri entryUri, Uri jsonUri) async {
-  await m.main(['--sdk-summary', '$sdkOutline', '$entryUri', '$jsonUri']);
+  final dart2jsEntry = ikgBenchmarks.resolve('dart2js.dart');
+  final dart2jsEdits = ikgBenchmarks.resolve('dart2js.edits.json');
+  await m
+      .main(['--sdk-summary', '$sdkOutline', '$dart2jsEntry', '$dart2jsEdits']);
+  await m.main([
+    '--sdk-summary',
+    '$sdkOutline',
+    '--mode=legacy',
+    '--implementation=driver',
+    '$dart2jsEntry',
+    '$dart2jsEdits'
+  ]);
+  await m.main([
+    '--sdk-summary',
+    '$sdkOutline',
+    '--mode=legacy',
+    '--implementation=minimal',
+    '$dart2jsEntry',
+    '$dart2jsEdits'
+  ]);
 }

tools/bots/try_benchmarks.sh

@@ -414,8 +414,10 @@ EOF
     echo '[{"name":"foo","edits":[["pkg/compiler/lib/src/dart2js.dart","2016","2017"],["pkg/compiler/lib/src/options.dart","2016","2017"]]}]' > appjit_train_edits.json
     out/ReleaseX64/dart --background-compilation=false --snapshot-kind=app-jit --snapshot=pkg/front_end/tool/incremental_perf.dart.appjit pkg/front_end/tool/incremental_perf.dart --target=vm --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/compiler/lib/src/dart2js.dart appjit_train_edits.json
     out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
-    echo '[{"name":"foo","edits":[["pkg/compiler/lib/src/dart2js.dart","2016","2017"],["pkg/compiler/lib/src/options.dart","2016","2017"]]}]' > appjit_train_edits.json && out/ReleaseX64/dart --background-compilation=false --snapshot-kind=app-jit --snapshot=pkg/front_end/tool/incremental_perf.dart.appjit pkg/front_end/tool/incremental_perf.dart --target=vm --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/compiler/lib/src/dart2js.dart appjit_train_edits.json
-    out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
+    out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --mode=strong --implementation=driver --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
+    out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --mode=legacy --implementation=driver --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
+    out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --mode=strong --implementation=minimal --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
+    out/ReleaseX64/dart --background-compilation=false pkg/front_end/tool/incremental_perf.dart.appjit --target=vm --mode=legacy --implementation=minimal --sdk-summary=out/ReleaseX64/vm_platform.dill --sdk-library-specification=sdk/lib/libraries.json pkg/front_end/benchmarks/ikg/hello.dart pkg/front_end/benchmarks/ikg/hello.edits.json
     cd ..
     rm -rf tmp
   else