tooling(bench_util): benching and profiling utilities (#10223)

Cargo.lock

@@ -1,5 +1,7 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
+version = 3
+
 [[package]]
 name = "Inflector"
 version = "0.11.4"
@@ -194,6 +196,15 @@ version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
 
+[[package]]
+name = "bench_util"
+version = "0.0.0"
+dependencies = [
+ "bencher",
+ "deno_core",
+ "tokio",
+]
+
 [[package]]
 name = "bencher"
 version = "0.1.5"
@@ -580,7 +591,7 @@ name = "deno_core"
 version = "0.84.0"
 dependencies = [
  "anyhow",
- "bencher",
+ "bench_util",
  "futures",
  "indexmap",
  "lazy_static",

Cargo.toml

@@ -7,6 +7,7 @@ members = [
   "runtime",
   "serde_v8",
   "test_plugin",
+  "bench_util",
   "test_util",
   "op_crates/crypto",
   "op_crates/fetch",

bench_util/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "bench_util"
+version = "0.0.0"
+authors = ["the Deno authors"]
+edition = "2018"
+description = "Bench and profiling utilities for deno crates"
+license = "MIT"
+readme = "README.md"
+repository = "https://github.com/denoland/deno"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bencher = "0.1"
+deno_core = { version = "0.84.0", path = "../core" }
+tokio = { version = "1.4.0", features = ["full"] }

bench_util/src/js_runtime.rs

@@ -0,0 +1,89 @@
+use bencher::Bencher;
+use deno_core::v8;
+use deno_core::JsRuntime;
+
+use crate::profiling::is_profiling;
+
+pub fn create_js_runtime(setup: impl FnOnce(&mut JsRuntime)) -> JsRuntime {
+  let mut rt = JsRuntime::new(Default::default());
+
+  // Setup bootstrap namespace
+  rt.execute("bootstrap", "globalThis.__bootstrap = {};")
+    .unwrap();
+
+  // Caller provided setup
+  setup(&mut rt);
+
+  // Init ops
+  rt.execute(
+    "init",
+    r#"
+      Deno.core.ops();
+      Deno.core.registerErrorClass('Error', Error);
+    "#,
+  )
+  .unwrap();
+
+  rt
+}
+
+fn loop_code(iters: u64, src: &str) -> String {
+  format!(r#"for(let i=0; i < {}; i++) {{ {} }}"#, iters, src,)
+}
+
+pub fn bench_js_sync(
+  b: &mut Bencher,
+  src: &str,
+  setup: impl FnOnce(&mut JsRuntime),
+) {
+  let mut runtime = create_js_runtime(setup);
+  let context = runtime.global_context();
+  let scope = &mut v8::HandleScope::with_context(runtime.v8_isolate(), context);
+
+  // Increase JS iterations if profiling for nicer flamegraphs
+  let inner_iters = 1000 * if is_profiling() { 10000 } else { 1 };
+  // Looped code
+  let looped_src = loop_code(inner_iters, src);
+
+  let code = v8::String::new(scope, looped_src.as_ref()).unwrap();
+  let script = v8::Script::compile(scope, code, None).unwrap();
+
+  // Run once if profiling, otherwise regular bench loop
+  if is_profiling() {
+    script.run(scope).unwrap();
+  } else {
+    b.iter(|| {
+      script.run(scope).unwrap();
+    });
+  }
+}
+
+pub fn bench_js_async(
+  b: &mut Bencher,
+  src: &str,
+  setup: impl FnOnce(&mut JsRuntime),
+) {
+  let mut runtime = create_js_runtime(setup);
+  let tokio_runtime = tokio::runtime::Builder::new_current_thread()
+    .enable_all()
+    .build()
+    .unwrap();
+
+  // Looped code
+  let looped = loop_code(1000, src);
+  let src = looped.as_ref();
+
+  if is_profiling() {
+    for _ in 0..10000 {
+      runtime.execute("inner_loop", src).unwrap();
+      let future = runtime.run_event_loop();
+      tokio_runtime.block_on(future).unwrap();
+    }
+  } else {
+    b.iter(|| {
+      runtime.execute("inner_loop", src).unwrap();
+      let future = runtime.run_event_loop();
+      tokio_runtime.block_on(future).unwrap();
+    });
+  }
+}

bench_util/src/lib.rs

@@ -0,0 +1,6 @@
+mod js_runtime;
+mod profiling;
+
+pub use bencher;
+pub use js_runtime::*;
+pub use profiling::*; // Exports bench_or_profile! macro

bench_util/src/profiling.rs

@@ -0,0 +1,81 @@
+use bencher::{DynBenchFn, StaticBenchFn, TestDescAndFn, TestOpts};
+
+pub fn is_profiling() -> bool {
+  std::env::var("PROFILING").is_ok()
+}
+
+#[macro_export]
+// Tweaked and copied from https://github.com/bluss/bencher/blob/master/macros.rs
+macro_rules! bench_or_profile {
+  ($($group_name:path),+) => {
+    fn main() {
+      use $crate::bencher::TestOpts;
+      use $crate::bencher::run_tests_console;
+      let mut test_opts = TestOpts::default();
+      // check to see if we should filter:
+      if let Some(arg) = ::std::env::args().skip(1).find(|arg| *arg != "--bench") {
+          test_opts.filter = Some(arg);
+      }
+      let mut benches = Vec::new();
+      $(
+          benches.extend($group_name());
+      )+
+
+      if $crate::is_profiling() {
+        // Run profling
+        $crate::run_profiles(&test_opts, benches);
+      } else {
+        // Run benches
+        run_tests_console(&test_opts, benches).unwrap();
+      }
+    }
+  };
+  ($($group_name:path,)+) => {
+      bench_or_profile!($($group_name),+);
+  };
+}
+
+pub fn run_profiles(opts: &TestOpts, tests: Vec<TestDescAndFn>) {
+  let tests = filter_tests(opts, tests);
+  // let decs = tests.iter().map(|t| t.desc.clone()).collect();
+
+  println!();
+  for b in tests {
+    println!("Profiling {}", b.desc.name);
+    run_profile(b);
+  }
+  println!();
+}
+
+fn run_profile(test: TestDescAndFn) {
+  match test.testfn {
+    DynBenchFn(bencher) => {
+      bencher::bench::run_once(|harness| bencher.run(harness));
+    }
+    StaticBenchFn(benchfn) => {
+      bencher::bench::run_once(|harness| benchfn(harness));
+    }
+  };
+}
+
+// Copied from https://github.com/bluss/bencher/blob/master/lib.rs
+fn filter_tests(
+  opts: &TestOpts,
+  tests: Vec<TestDescAndFn>,
+) -> Vec<TestDescAndFn> {
+  let mut filtered = tests;
+
+  // Remove tests that don't match the test filter
+  filtered = match opts.filter {
+    None => filtered,
+    Some(ref filter) => filtered
+      .into_iter()
+      .filter(|test| test.desc.name.contains(&filter[..]))
+      .collect(),
+  };
+
+  // Sort the tests alphabetically
+  filtered.sort_by(|t1, t2| t1.desc.name.cmp(&t2.desc.name));
+
+  filtered
+}

core/Cargo.toml

@@ -34,7 +34,7 @@ path = "examples/http_bench_json_ops.rs"
 # These dependencies are only used for the 'http_bench_*_ops' examples.
 [dev-dependencies]
 tokio = { version = "1.4.0", features = ["full"] }
-bencher = "0.1"
+bench_util = { version = "0.0.0", path = "../bench_util" }
 
 [[bench]]
 name = "op_baseline"

core/benches/op_baseline.rs

@@ -1,38 +1,25 @@
-use bencher::{benchmark_group, benchmark_main, Bencher};
-
 use deno_core::error::AnyError;
 use deno_core::op_async;
 use deno_core::op_sync;
 use deno_core::serialize_op_result;
-use deno_core::v8;
 use deno_core::JsRuntime;
 use deno_core::Op;
 use deno_core::OpState;
 use deno_core::ZeroCopyBuf;
 
+use bench_util::bench_or_profile;
+use bench_util::bencher::{benchmark_group, Bencher};
+use bench_util::{bench_js_async, bench_js_sync};
+
 use std::cell::RefCell;
 use std::rc::Rc;
 
-fn create_js_runtime() -> JsRuntime {
-  let mut runtime = JsRuntime::new(Default::default());
-  runtime.register_op("pi_json", op_sync(|_, _: (), _| Ok(314159)));
-  runtime.register_op("pi_async", op_async(op_pi_async));
-  runtime.register_op("nop", |state, _, _| {
+fn setup(rt: &mut JsRuntime) {
+  rt.register_op("pi_json", op_sync(|_, _: (), _| Ok(314159)));
+  rt.register_op("pi_async", op_async(op_pi_async));
+  rt.register_op("nop", |state, _, _| {
     Op::Sync(serialize_op_result(Ok(9), state))
   });
-
-  // Init ops
-  runtime
-    .execute(
-      "init",
-      r#"
-      Deno.core.ops();
-      Deno.core.registerErrorClass('Error', Error);
-    "#,
-    )
-    .unwrap();
-
-  runtime
 }
 
 // this is a function since async closures aren't stable
@@ -44,57 +31,21 @@ async fn op_pi_async(
   Ok(314159)
 }
 
-pub fn bench_runtime_js(b: &mut Bencher, src: &str) {
-  let mut runtime = create_js_runtime();
-  let context = runtime.global_context();
-  let scope = &mut v8::HandleScope::with_context(runtime.v8_isolate(), context);
-  let code = v8::String::new(scope, src).unwrap();
-  let script = v8::Script::compile(scope, code, None).unwrap();
-  b.iter(|| {
-    script.run(scope).unwrap();
-  });
-}
-
-pub fn bench_runtime_js_async(b: &mut Bencher, src: &str) {
-  let mut runtime = create_js_runtime();
-  let tokio_runtime = tokio::runtime::Builder::new_current_thread()
-    .enable_all()
-    .build()
-    .unwrap();
-
-  b.iter(|| {
-    runtime.execute("inner_loop", src).unwrap();
-    let future = runtime.run_event_loop();
-    tokio_runtime.block_on(future).unwrap();
-  });
-}
-
 fn bench_op_pi_json(b: &mut Bencher) {
-  bench_runtime_js(
-    b,
-    r#"for(let i=0; i < 1e3; i++) {
-      Deno.core.opSync("pi_json", null);
-    }"#,
-  );
+  bench_js_sync(b, r#"Deno.core.opSync("pi_json");"#, setup);
 }
 
 fn bench_op_nop(b: &mut Bencher) {
-  bench_runtime_js(
+  bench_js_sync(
     b,
-    r#"for(let i=0; i < 1e3; i++) {
-      Deno.core.dispatchByName("nop", null, null, null);
-    }"#,
+    r#"Deno.core.dispatchByName("nop", null, null, null);"#,
+    setup,
   );
 }
 
 fn bench_op_async(b: &mut Bencher) {
-  bench_runtime_js_async(
-    b,
-    r#"for(let i=0; i < 1e3; i++) {
-      Deno.core.opAsync("pi_async", null);
-    }"#,
-  );
+  bench_js_async(b, r#"Deno.core.opAsync("pi_async");"#, setup);
 }
 
 benchmark_group!(benches, bench_op_pi_json, bench_op_nop, bench_op_async);
-benchmark_main!(benches);
+bench_or_profile!(benches);