[vm/benchmarks] Add IsolateSpawn spawn latency and memory benchmarks.

The benchmark spawns an isolate that compiles hello world app with dart2js, measures time it takes for spawned isolate to get up and running, measures delta rss when isolate is spawned.

This also adds analysis_options.yaml that helps with keeping the code lint-free.

Change-Id: I5f1ffa9706766cd00bf1ea3fdad76957952de8a1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/119538
Reviewed-by: Jonas Termansen <sortie@google.com>
Commit-Queue: Alexander Aprelev <aam@google.com>

benchmarks/IsolateSpawn/dart/IsolateSpawn.dart

@@ -0,0 +1,182 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+import 'dart:isolate';
+import 'dart:math';
+
+import 'package:meta/meta.dart';
+
+import 'package:compiler/src/dart2js.dart' as dart2js_main;
+
+class SpawnLatencyAndMemory {
+  SpawnLatencyAndMemory(this.name);
+
+  Future<ResultMessageLatencyAndMemory> run() async {
+    final completerResult = Completer();
+    final receivePort = ReceivePort()..listen(completerResult.complete);
+    final Completer<DateTime> isolateExitedCompleter = Completer<DateTime>();
+    final onExitReceivePort = ReceivePort()
+      ..listen((_) {
+        isolateExitedCompleter.complete(DateTime.now());
+      });
+    final DateTime beforeSpawn = DateTime.now();
+    await Isolate.spawn(
+        isolateCompiler,
+        StartMessageLatencyAndMemory(
+            receivePort.sendPort, beforeSpawn, ProcessInfo.currentRss),
+        onExit: onExitReceivePort.sendPort,
+        onError: onExitReceivePort.sendPort);
+    final DateTime afterSpawn = DateTime.now();
+
+    final ResultMessageLatencyAndMemory result = await completerResult.future;
+    receivePort.close();
+    final DateTime isolateExited = await isolateExitedCompleter.future;
+    result.timeToExitUs = isolateExited.difference(beforeSpawn).inMicroseconds;
+    result.timeToIsolateSpawnUs =
+        afterSpawn.difference(beforeSpawn).inMicroseconds;
+    onExitReceivePort.close();
+
+    return result;
+  }
+
+  Future<AggregatedResultMessageLatencyAndMemory> measureFor(
+      int minimumMillis) async {
+    final minimumMicros = minimumMillis * 1000;
+    final watch = Stopwatch()..start();
+    final Metric toAfterIsolateSpawnUs = LatencyMetric("${name}ToAfterSpawn");
+    final Metric toStartRunningCodeUs = LatencyMetric("${name}ToStartRunning");
+    final Metric toFinishRunningCodeUs =
+        LatencyMetric("${name}ToFinishRunning");
+    final Metric toExitUs = LatencyMetric("${name}ToExit");
+    final Metric deltaRss = MemoryMetric("${name}Delta");
+    while (watch.elapsedMicroseconds < minimumMicros) {
+      final ResultMessageLatencyAndMemory result = await run();
+      toAfterIsolateSpawnUs.add(result.timeToIsolateSpawnUs);
+      toStartRunningCodeUs.add(result.timeToStartRunningCodeUs);
+      toFinishRunningCodeUs.add(result.timeToFinishRunningCodeUs);
+      toExitUs.add(result.timeToExitUs);
+      deltaRss.add(result.deltaRss);
+    }
+    return AggregatedResultMessageLatencyAndMemory(toAfterIsolateSpawnUs,
+        toStartRunningCodeUs, toFinishRunningCodeUs, toExitUs, deltaRss);
+  }
+
+  Future<AggregatedResultMessageLatencyAndMemory> measure() async {
+    await measureFor(500); // warm-up
+    return measureFor(4000); // actual measurement
+  }
+
+  Future<void> report() async {
+    final AggregatedResultMessageLatencyAndMemory result = await measure();
+    print(result);
+  }
+
+  final String name;
+  RawReceivePort receivePort;
+}
+
+class Metric {
+  Metric({@required this.prefix, @required this.suffix});
+
+  void add(int value) {
+    if (value > max) {
+      max = value;
+    }
+    sum += value;
+    sumOfSquares += value * value;
+    count++;
+  }
+
+  double _average() => sum / count;
+  double _rms() => sqrt(sumOfSquares / count);
+
+  toString() => "$prefix): ${_average()}$suffix\n"
+      "${prefix}Max): $max$suffix\n"
+      "${prefix}RMS): ${_rms()}$suffix";
+
+  final String prefix;
+  final String suffix;
+  int max = 0;
+  double sum = 0;
+  double sumOfSquares = 0;
+  int count = 0;
+}
+
+class LatencyMetric extends Metric {
+  LatencyMetric(String name) : super(prefix: "${name}(Latency", suffix: " us.");
+}
+
+class MemoryMetric extends Metric {
+  MemoryMetric(String name) : super(prefix: "${name}Rss(MemoryUse", suffix: "");
+
+  toString() => "$prefix): ${_average()}$suffix\n";
+}
+
+class StartMessageLatencyAndMemory {
+  StartMessageLatencyAndMemory(this.sendPort, this.spawned, this.rss);
+
+  final SendPort sendPort;
+  final DateTime spawned;
+  final int rss;
+}
+
+class ResultMessageLatencyAndMemory {
+  ResultMessageLatencyAndMemory(
+      {this.timeToStartRunningCodeUs,
+      this.timeToFinishRunningCodeUs,
+      this.deltaRss});
+
+  final int timeToStartRunningCodeUs;
+  final int timeToFinishRunningCodeUs;
+  final int deltaRss;
+
+  int timeToIsolateSpawnUs;
+  int timeToExitUs;
+}
+
+class AggregatedResultMessageLatencyAndMemory {
+  AggregatedResultMessageLatencyAndMemory(
+    this.toAfterIsolateSpawnUs,
+    this.toStartRunningCodeUs,
+    this.toFinishRunningCodeUs,
+    this.toExitUs,
+    this.deltaRss,
+  );
+
+  String toString() => """$toAfterIsolateSpawnUs
+$toStartRunningCodeUs
+$toFinishRunningCodeUs
+$toExitUs
+$deltaRss""";
+
+  final Metric toAfterIsolateSpawnUs;
+  final Metric toStartRunningCodeUs;
+  final Metric toFinishRunningCodeUs;
+  final Metric toExitUs;
+  final Metric deltaRss;
+}
+
+Future<void> isolateCompiler(StartMessageLatencyAndMemory start) async {
+  final DateTime timeRunningCodeUs = DateTime.now();
+  await runZoned(
+      () => dart2js_main.internalMain(<String>[
+            "benchmarks/IsolateSpawn/dart/helloworld.dart",
+            '--libraries-spec=sdk/lib/libraries.json'
+          ]),
+      zoneSpecification: ZoneSpecification(
+          print: (Zone self, ZoneDelegate parent, Zone zone, String line) {}));
+  final DateTime timeFinishRunningCodeUs = DateTime.now();
+  start.sendPort.send(ResultMessageLatencyAndMemory(
+      timeToStartRunningCodeUs:
+          timeRunningCodeUs.difference(start.spawned).inMicroseconds,
+      timeToFinishRunningCodeUs:
+          timeFinishRunningCodeUs.difference(start.spawned).inMicroseconds,
+      deltaRss: ProcessInfo.currentRss - start.rss));
+}
+
+Future<void> main() async {
+  await SpawnLatencyAndMemory("IsolateSpawn.Dart2JS").report();
+}

benchmarks/IsolateSpawn/dart/helloworld.dart

@@ -0,0 +1,3 @@
+main() {
+  print('Hello, world!');
+}

benchmarks/analysis_options.yaml

@@ -0,0 +1,83 @@
+include: package:pedantic/analysis_options.yaml
+#analyzer:
+#  strong-mode:
+#    implicit-casts: false
+linter:
+  rules:
+    #- always_declare_return_types
+    #- annotate_overrides
+    - avoid_empty_else
+    - avoid_function_literals_in_foreach_calls
+    - avoid_init_to_null
+    - avoid_null_checks_in_equality_operators
+    - avoid_relative_lib_imports
+    - avoid_renaming_method_parameters
+    - avoid_return_types_on_setters
+    - avoid_returning_null
+    - avoid_returning_null_for_future
+    - avoid_shadowing_type_parameters
+    - avoid_types_as_parameter_names
+    - avoid_unused_constructor_parameters
+    - await_only_futures
+    - camel_case_types
+    - cancel_subscriptions
+    - comment_references
+    #- constant_identifier_names
+    - control_flow_in_finally
+    - directives_ordering
+    - empty_catches
+    - empty_constructor_bodies
+    - empty_statements
+    - hash_and_equals
+    - implementation_imports
+    - invariant_booleans
+    - iterable_contains_unrelated_type
+    - library_names
+    - library_prefixes
+    - list_remove_unrelated_type
+    - literal_only_boolean_expressions
+    - no_adjacent_strings_in_list
+    - no_duplicate_case_values
+    #- non_constant_identifier_names
+    - null_closures
+    #- omit_local_variable_types
+    #- only_throw_errors
+    - overridden_fields
+    - package_api_docs
+    - package_names
+    - package_prefixed_library_names
+    - prefer_adjacent_string_concatenation
+    - prefer_collection_literals
+    - prefer_conditional_assignment
+    - prefer_const_constructors
+    - prefer_contains
+    - prefer_equal_for_default_values
+    - prefer_generic_function_type_aliases
+    - prefer_final_fields
+    - prefer_final_locals
+    - prefer_initializing_formals
+    - prefer_interpolation_to_compose_strings
+    - prefer_is_empty
+    - prefer_is_not_empty
+    #- prefer_single_quotes
+    #- prefer_typing_uninitialized_variables
+    - recursive_getters
+    - slash_for_doc_comments
+    - test_types_in_equals
+    - throw_in_finally
+    - type_init_formals
+    - unawaited_futures
+    - unnecessary_await_in_return
+    - unnecessary_brace_in_string_interps
+    - unnecessary_const
+    - unnecessary_getters_setters
+    - unnecessary_lambdas
+    - unnecessary_new
+    - unnecessary_null_aware_assignments
+    - unnecessary_parenthesis
+    - unnecessary_statements
+    #- unnecessary_this
+    - unrelated_type_equality_checks
+    - use_function_type_syntax_for_parameters
+    - use_rethrow_when_possible
+    - valid_regexps

tools/bots/try_benchmarks.sh

@@ -89,6 +89,7 @@ for command; do
       -- \
       third_party/d8/linux/ia32/natives_blob.bin \
       third_party/d8/linux/ia32/snapshot_blob.bin \
+      out/ReleaseIA32/dart2js_platform.dill \
       out/ReleaseIA32/vm_outline_strong.dill \
       out/ReleaseIA32/vm_platform_strong.dill \
       out/ReleaseIA32/gen/kernel_service.dill \
@@ -174,6 +175,7 @@ for command; do
       -- \
       third_party/d8/linux/ia32/natives_blob.bin \
       third_party/d8/linux/ia32/snapshot_blob.bin \
+      out/ReleaseIA32/dart2js_platform.dill \
       out/ReleaseIA32/vm_outline_strong.dill \
       out/ReleaseIA32/vm_platform_strong.dill \
       out/ReleaseIA32/gen/kernel_service.dill \
@@ -254,6 +256,7 @@ EOF
       -- \
       third_party/d8/linux/x64/natives_blob.bin \
       third_party/d8/linux/x64/snapshot_blob.bin \
+      out/ReleaseX64/dart2js_platform.dill \
       out/ReleaseX64/vm_outline_strong.dill \
       out/ReleaseX64/vm_platform_strong.dill \
       out/ReleaseX64/gen/kernel_service.dill \
@@ -358,6 +361,7 @@ EOF
       -- \
       third_party/d8/linux/x64/natives_blob.bin \
       third_party/d8/linux/x64/snapshot_blob.bin \
+      out/ReleaseX64/dart2js_platform.dill \
       out/ReleaseX64/vm_outline_strong.dill \
       out/ReleaseX64/vm_platform_strong.dill \
       out/ReleaseX64/gen/kernel_service.dill \