fix(bench): Fix group header printing logic + don't filter out the warmup benchmark (#23083)

Fixes #23053.
Two small bugs here:
- the existing condition for printing out the group header was broken.
it worked in the reproducer (in the issue above) without filtering only
by accident, due to setting `self.has_ungrouped = true` once we see the
warmup bench. Knowing that we sort benchmarks to put ungrouped benches
first, there are only two cases: 1) we are starting the first group 2)
we are ending the previous group and starting a new group
- when you passed `--filter` we were applying that filter to the warmup
bench (which is not visible to users), so we suffered from jit bias if
you were filtering (unless your filter was `<warmup>`)

TLDR;

Running
```bash
deno bench main.js --filter="G"
```
```js
// main.js
Deno.bench({
  group: "G1",
  name: "G1-A",
  fn() {},
});

Deno.bench({
  group: "G1",
  name: "G1-B",
  fn() {},
});
```


Before this PR:
```
benchmark      time (avg)        iter/s             (min … max)       p75       p99      p995
--------------------------------------------------------------- -----------------------------

G1-A          303.52 ps/iter3,294,726,102.1     (254.2 ps … 7.8 ns) 287.5 ps 391.7 ps 437.5 ps
G1-B             3.8 ns/iter 263,360,635.9     (2.24 ns … 8.36 ns) 3.84 ns 4.73 ns 4.94 ns

summary
  G1-A
   12.51x faster than G1-B
```

After this PR:
```
benchmark      time (avg)        iter/s             (min … max)       p75       p99      p995
--------------------------------------------------------------- -----------------------------

group G1
G1-A            3.85 ns/iter 259,822,096.0     (2.42 ns … 9.03 ns) 3.83 ns 4.62 ns 4.83 ns
G1-B            3.84 ns/iter 260,458,274.5     (3.55 ns … 7.05 ns) 3.83 ns 4.45 ns 4.7 ns

summary
  G1-B
   1x faster than G1-A
```
This commit is contained in:
Nathan Whitaker 2024-03-26 09:19:24 -07:00 committed by GitHub
parent 89aa6d5cf6
commit a2a537e196
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 31 additions and 11 deletions

View File

@ -233,7 +233,7 @@ async fn bench_specifier_inner(
let benchmarks = if used_only { only } else { no_only };
let mut benchmarks = benchmarks
.into_iter()
.filter(|(d, _)| filter.includes(&d.name) && !d.ignore)
.filter(|(d, _)| d.warmup || filter.includes(&d.name) && !d.ignore)
.collect::<Vec<_>>();
let mut groups = IndexSet::<Option<String>>::new();
// make sure ungrouped benchmarks are placed above grouped

View File

@ -99,7 +99,6 @@ impl BenchReporter for JsonReporter {
pub struct ConsoleReporter {
name: String,
show_output: bool,
has_ungrouped: bool,
group: Option<String>,
baseline: bool,
group_measurements: Vec<(BenchDescription, BenchStats)>,
@ -114,7 +113,6 @@ impl ConsoleReporter {
options: None,
baseline: false,
name: String::new(),
has_ungrouped: false,
group_measurements: Vec::new(),
}
}
@ -173,18 +171,11 @@ impl BenchReporter for ConsoleReporter {
self.name = desc.name.clone();
match &desc.group {
None => {
self.has_ungrouped = true;
}
None => {}
Some(group) => {
if self.group.is_none() || group != self.group.as_ref().unwrap() {
self.report_group_summary();
}
if (self.group.is_none() && self.has_ungrouped)
|| (self.group.is_some() && self.group_measurements.is_empty())
{
println!("{} {}", colors::gray("group"), colors::green(group));
}

View File

@ -0,0 +1,5 @@
// Regression test for https://github.com/denoland/deno/issues/23053
{
"args": "bench main.ts --filter=G",
"output": "main.out"
}

View File

@ -0,0 +1,13 @@
Check [WILDCARD]
cpu: [WILDCARD]
runtime: [WILDCARD]
[WILDCARD]
benchmark time (avg) iter/s (min … max) p75 p99 p995
--------------------------------------------------------------- -----------------------------
group G1
G1-B [WILDCARD]
group G2
G2-B [WILDCARD]

View File

@ -0,0 +1,11 @@
Deno.bench({
group: "G1",
name: "G1-B",
fn() {},
});
Deno.bench({
group: "G2",
name: "G2-B",
fn() {},
});