[benchmark] Add top level benchmarks directory.

Benchmarks are checked in as benchmarks/<Benchmark>/dart/<Benchmark>.dart.
This scheme is compatible with our existing benchmarking infrastructure
and will aid migrating the benchmarks with minimal breakage.

This change adds an Example benchmark to show how it is done.

The benchmarks directory is now added to the Dart benchmarking builds.

Change-Id: I25971ba3b219194fa9cfea6b938372d877477e28
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/108414
Commit-Queue: Jonas Termansen <sortie@google.com>
Reviewed-by: Daco Harkes <dacoharkes@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

.gitignore

@@ -3,7 +3,6 @@
 /.project
 /Makefile
 /base
-/benchmarks
 /buildtools
 /ipch
 /out

.packages

@@ -17,6 +17,7 @@ args:third_party/pkg/args/lib
 async:third_party/pkg/async/lib
 async_helper:pkg/async_helper/lib
 bazel_worker:third_party/pkg/bazel_worker/lib
+benchmark_harness:third_party/pkg/benchmark_harness/lib
 boolean_selector:third_party/pkg/boolean_selector/lib
 build_integration:pkg/build_integration/lib
 charcode:third_party/pkg/charcode/lib

DEPS

@@ -55,6 +55,7 @@ vars = {
   "args_tag": "1.4.4",
   "async_tag": "2.0.8",
   "bazel_worker_tag": "bazel_worker-v0.1.20",
+  "benchmark_harness_tag": "81641290dea44c34138a109a37e215482f405f81",
   "boolean_selector_tag" : "1.0.4",
   "boringssl_gen_rev": "bbf52f18f425e29b1185f2f6753bec02ed8c5880",
   "boringssl_rev" : "702e2b6d3831486535e958f262a05c75a5cb312e",
@@ -236,6 +237,9 @@ deps = {
       Var("dart_git") + "async.git" + "@" + Var("async_tag"),
   Var("dart_root") + "/third_party/pkg/bazel_worker":
       Var("dart_git") + "bazel_worker.git" + "@" + Var("bazel_worker_tag"),
+  Var("dart_root") + "/third_party/pkg/benchmark_harness":
+      Var("dart_git") + "benchmark_harness.git" + "@" +
+      Var("benchmark_harness_tag"),
   Var("dart_root") + "/third_party/pkg/boolean_selector":
       Var("dart_git") + "boolean_selector.git" +
       "@" + Var("boolean_selector_tag"),

benchmarks/Example/dart/Example.dart

@@ -0,0 +1,22 @@
+// Copyright (c) 2019, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+class Example extends BenchmarkBase {
+  const Example() : super("Example");
+
+  // The benchmark code.
+  void run() {}
+
+  // Not measured setup code executed prior to the benchmark runs.
+  void setup() {}
+
+  // Not measures teardown code executed after the benchark runs.
+  void teardown() {}
+}
+
+main() {
+  const Example().report();
+}

tools/bots/try_benchmarks.sh

@@ -105,6 +105,7 @@ for command; do
       pkg \
       runtime/bin \
       runtime/lib \
+      benchmarks \
       || (rm -f linux-ia32_profile.tar.gz; exit 1)
     strip -w \
       -K 'kDartVmSnapshotData' \
@@ -188,6 +189,7 @@ for command; do
       pkg \
       runtime/bin \
       runtime/lib \
+      benchmarks \
       || (rm -f linux-ia32.tar.gz; exit 1)
   elif [ "$command" = linux-ia32-benchmark ]; then
     rm -rf tmp
@@ -219,6 +221,7 @@ EOF
     out/ReleaseIA32/run_vm_tests InitialRSS
     out/ReleaseIA32/run_vm_tests GenKernelKernelLoadKernel
     out/ReleaseIA32/run_vm_tests KernelServiceCompileAll
+    out/ReleaseIA32/dart --profile-period=10000 --packages=.packages benchmarks/Example/dart/Example.dart
     cd ..
     rm -rf tmp
   elif [ "$command" = linux-x64-build ] ||
@@ -264,6 +267,7 @@ EOF
       pkg \
       runtime/bin \
       runtime/lib \
+      benchmarks \
       || (rm -f linux-x64_profile.tar.gz; exit 1)
     strip -w \
       -K 'kDartVmSnapshotData' \
@@ -366,6 +370,7 @@ EOF
       pkg \
       runtime/bin \
       runtime/lib \
+      benchmarks \
       || (rm -f linux-x64.tar.gz; exit 1)
   elif [ "$command" = linux-x64-benchmark ] ||
        [ "$command" = linux-x64-bytecode-benchmark ]; then
@@ -403,6 +408,7 @@ EOF
     out/ReleaseX64/run_vm_tests InitialRSS
     out/ReleaseX64/run_vm_tests GenKernelKernelLoadKernel
     out/ReleaseX64/run_vm_tests KernelServiceCompileAll
+    out/ReleaseX64/dart --profile-period=10000 --packages=.packages benchmarks/Example/dart/Example.dart
     cd ..
     rm -rf tmp
   else