feat: Stabilize Deno.bench() and 'deno bench' subcommand (#16485)

This commit is contained in:
Bartek Iwańczuk 2022-11-11 00:22:14 +01:00 committed by GitHub
parent 79aa3124a8
commit 5be8c96ae8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 264 additions and 310 deletions

View file

@ -14,13 +14,11 @@ use std::fmt;
const MAX_SOURCE_LINE_LENGTH: usize = 150;
const UNSTABLE_DENO_PROPS: &[&str] = &[
"BenchDefinition",
"CreateHttpClientOptions",
"DatagramConn",
"HttpClient",
"UnixConnectOptions",
"UnixListenOptions",
"bench",
"connect",
"createHttpClient",
"kill",

View file

@ -931,6 +931,242 @@ declare namespace Deno {
fn: (t: TestContext) => void | Promise<void>,
): void;
/**
* The interface for defining a benchmark test using {@linkcode Deno.bench}.
*
* @category Testing
*/
export interface BenchDefinition {
/** The test function which will be benchmarked. */
fn: () => void | Promise<void>;
/** The name of the test, which will be used in displaying the results. */
name: string;
/** If truthy, the benchmark test will be ignored/skipped. */
ignore?: boolean;
/** Group name for the benchmark.
*
* Grouped benchmarks produce a group time summary, where the difference
* in performance between each test of the group is compared. */
group?: string;
/** Benchmark should be used as the baseline for other benchmarks.
*
* If there are multiple baselines in a group, the first one is used as the
* baseline. */
baseline?: boolean;
/** If at least one bench has `only` set to true, only run benches that have
* `only` set to `true` and fail the bench suite. */
only?: boolean;
/** Ensure the bench case does not prematurely cause the process to exit,
* for example via a call to {@linkcode Deno.exit}. Defaults to `true`. */
sanitizeExit?: boolean;
/** Specifies the permissions that should be used to run the bench.
*
* Set this to `"inherit"` to keep the calling thread's permissions.
*
* Set this to `"none"` to revoke all permissions.
*
* Defaults to "inherit".
*/
permissions?: Deno.PermissionOptions;
}
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench({
* name: "example test",
* fn() {
* assertEquals("world", "world");
* },
* });
*
* Deno.bench({
* name: "example ignored test",
* ignore: Deno.build.os === "windows",
* fn() {
* // This test is ignored only on Windows machines
* },
* });
*
* Deno.bench({
* name: "example async test",
* async fn() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* });
* ```
*
* @category Testing
*/
export function bench(t: BenchDefinition): void;
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench("My test description", () => {
* assertEquals("hello", "hello");
* });
*
* Deno.bench("My async test description", async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* });
* ```
*
* @category Testing
*/
export function bench(
name: string,
fn: () => void | Promise<void>,
): void;
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(function myTestName() {
* assertEquals("hello", "hello");
* });
*
* Deno.bench(async function myOtherTestName() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* });
* ```
*
* @category Testing
*/
export function bench(fn: () => void | Promise<void>): void;
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* "My test description",
* { permissions: { read: true } },
* () => {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* "My async test description",
* { permissions: { read: false } },
* async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
name: string,
options: Omit<BenchDefinition, "fn" | "name">,
fn: () => void | Promise<void>,
): void;
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* { name: "My test description", permissions: { read: true } },
* () => {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* { name: "My async test description", permissions: { read: false } },
* async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
options: Omit<BenchDefinition, "fn">,
fn: () => void | Promise<void>,
): void;
/**
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* { permissions: { read: true } },
* function myTestName() {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* { permissions: { read: false } },
* async function myOtherTestName() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
options: Omit<BenchDefinition, "fn" | "name">,
fn: () => void | Promise<void>,
): void;
/** Exit the Deno process with optional exit code.
*
* If no exit code is supplied then Deno will exit with return code of `0`.

View file

@ -6,249 +6,6 @@
declare namespace Deno {
export {}; // stop default export type behavior
/** **UNSTABLE**: New API, yet to be vetted.
*
* The interface for defining a benchmark test using {@linkcode Deno.bench}.
*
* @category Testing
*/
export interface BenchDefinition {
/** The test function which will be benchmarked. */
fn: () => void | Promise<void>;
/** The name of the test, which will be used in displaying the results. */
name: string;
/** If truthy, the benchmark test will be ignored/skipped. */
ignore?: boolean;
/** Group name for the benchmark.
*
* Grouped benchmarks produce a group time summary, where the difference
* in performance between each test of the group is compared. */
group?: string;
/** Benchmark should be used as the baseline for other benchmarks.
*
* If there are multiple baselines in a group, the first one is used as the
* baseline. */
baseline?: boolean;
/** If at least one bench has `only` set to true, only run benches that have
* `only` set to `true` and fail the bench suite. */
only?: boolean;
/** Ensure the bench case does not prematurely cause the process to exit,
* for example via a call to {@linkcode Deno.exit}. Defaults to `true`. */
sanitizeExit?: boolean;
/** Specifies the permissions that should be used to run the bench.
*
* Set this to `"inherit"` to keep the calling thread's permissions.
*
* Set this to `"none"` to revoke all permissions.
*
* Defaults to "inherit".
*/
permissions?: Deno.PermissionOptions;
}
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench({
* name: "example test",
* fn() {
* assertEquals("world", "world");
* },
* });
*
* Deno.bench({
* name: "example ignored test",
* ignore: Deno.build.os === "windows",
* fn() {
* // This test is ignored only on Windows machines
* },
* });
*
* Deno.bench({
* name: "example async test",
* async fn() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* });
* ```
*
* @category Testing
*/
export function bench(t: BenchDefinition): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench("My test description", () => {
* assertEquals("hello", "hello");
* });
*
* Deno.bench("My async test description", async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* });
* ```
*
* @category Testing
*/
export function bench(
name: string,
fn: () => void | Promise<void>,
): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(function myTestName() {
* assertEquals("hello", "hello");
* });
*
* Deno.bench(async function myOtherTestName() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* });
* ```
*
* @category Testing
*/
export function bench(fn: () => void | Promise<void>): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* "My test description",
* { permissions: { read: true } },
* () => {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* "My async test description",
* { permissions: { read: false } },
* async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
name: string,
options: Omit<BenchDefinition, "fn" | "name">,
fn: () => void | Promise<void>,
): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* { name: "My test description", permissions: { read: true } },
* () => {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* { name: "My async test description", permissions: { read: false } },
* async () => {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
options: Omit<BenchDefinition, "fn">,
fn: () => void | Promise<void>,
): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Register a benchmark test which will be run when `deno bench` is used on
* the command line and the containing module looks like a bench module.
*
* If the test function (`fn`) returns a promise or is async, the test runner
* will await resolution to consider the test complete.
*
* ```ts
* import { assertEquals } from "https://deno.land/std/testing/asserts.ts";
*
* Deno.bench(
* { permissions: { read: true } },
* function myTestName() {
* assertEquals("hello", "hello");
* }
* );
*
* Deno.bench(
* { permissions: { read: false } },
* async function myOtherTestName() {
* const decoder = new TextDecoder("utf-8");
* const data = await Deno.readFile("hello_world.txt");
* assertEquals(decoder.decode(data), "Hello world");
* }
* );
* ```
*
* @category Testing
*/
export function bench(
options: Omit<BenchDefinition, "fn" | "name">,
fn: () => void | Promise<void>,
): void;
/** **UNSTABLE**: New API, yet to be vetted.
*
* Retrieve the process umask. If `mask` is provided, sets the process umask.

View file

@ -21,7 +21,6 @@ use uuid::Uuid;
pub fn init(
sender: UnboundedSender<BenchEvent>,
filter: TestFilter,
unstable: bool,
) -> Extension {
Extension::builder()
.ops(vec![
@ -31,36 +30,15 @@ pub fn init(
op_register_bench::decl(),
op_dispatch_bench_event::decl(),
op_bench_now::decl(),
op_bench_check_unstable::decl(),
])
.state(move |state| {
state.put(sender.clone());
state.put(filter.clone());
state.put(Unstable(unstable));
Ok(())
})
.build()
}
pub struct Unstable(pub bool);
fn check_unstable(state: &OpState, api_name: &str) {
let unstable = state.borrow::<Unstable>();
if !unstable.0 {
eprintln!(
"Unstable API '{}'. The --unstable flag must be provided.",
api_name
);
std::process::exit(70);
}
}
#[op]
fn op_bench_check_unstable(state: &mut OpState) {
check_unstable(state, "Deno.bench");
}
#[derive(Clone)]
struct PermissionsHolder(Uuid, Permissions);

View file

@ -4,165 +4,158 @@ use crate::itest;
use deno_core::url::Url;
use test_util as util;
itest!(requires_unstable {
args: "bench bench/requires_unstable.js",
exit_code: 70,
output: "bench/requires_unstable.out",
});
itest!(overloads {
args: "bench --unstable bench/overloads.ts",
args: "bench bench/overloads.ts",
exit_code: 0,
output: "bench/overloads.out",
});
itest!(meta {
args: "bench --unstable bench/meta.ts",
args: "bench bench/meta.ts",
exit_code: 0,
output: "bench/meta.out",
});
itest!(pass {
args: "bench --unstable bench/pass.ts",
args: "bench bench/pass.ts",
exit_code: 0,
output: "bench/pass.out",
});
itest!(ignore {
args: "bench --unstable bench/ignore.ts",
args: "bench bench/ignore.ts",
exit_code: 0,
output: "bench/ignore.out",
});
itest!(ignore_permissions {
args: "bench --unstable bench/ignore_permissions.ts",
args: "bench bench/ignore_permissions.ts",
exit_code: 0,
output: "bench/ignore_permissions.out",
});
itest!(fail {
args: "bench --unstable bench/fail.ts",
args: "bench bench/fail.ts",
exit_code: 1,
output: "bench/fail.out",
});
itest!(collect {
args: "bench --unstable --ignore=bench/collect/ignore bench/collect",
args: "bench --ignore=bench/collect/ignore bench/collect",
exit_code: 0,
output: "bench/collect.out",
});
itest!(load_unload {
args: "bench --unstable bench/load_unload.ts",
args: "bench bench/load_unload.ts",
exit_code: 0,
output: "bench/load_unload.out",
});
itest!(interval {
args: "bench --unstable bench/interval.ts",
args: "bench bench/interval.ts",
exit_code: 0,
output: "bench/interval.out",
});
itest!(quiet {
args: "bench --unstable --quiet bench/quiet.ts",
args: "bench --quiet bench/quiet.ts",
exit_code: 0,
output: "bench/quiet.out",
});
itest!(only {
args: "bench --unstable bench/only.ts",
args: "bench bench/only.ts",
exit_code: 1,
output: "bench/only.out",
});
itest!(multifile_summary {
args: "bench --unstable bench/group_baseline.ts bench/pass.ts bench/group_baseline.ts",
args: "bench bench/group_baseline.ts bench/pass.ts bench/group_baseline.ts",
exit_code: 0,
output: "bench/multifile_summary.out",
});
itest!(no_check {
args: "bench --unstable --no-check bench/no_check.ts",
args: "bench --no-check bench/no_check.ts",
exit_code: 1,
output: "bench/no_check.out",
});
itest!(allow_all {
args: "bench --unstable --allow-all bench/allow_all.ts",
args: "bench --allow-all bench/allow_all.ts",
exit_code: 0,
output: "bench/allow_all.out",
});
itest!(allow_none {
args: "bench --unstable bench/allow_none.ts",
args: "bench bench/allow_none.ts",
exit_code: 1,
output: "bench/allow_none.out",
});
itest!(exit_sanitizer {
args: "bench --unstable bench/exit_sanitizer.ts",
args: "bench bench/exit_sanitizer.ts",
output: "bench/exit_sanitizer.out",
exit_code: 1,
});
itest!(clear_timeout {
args: "bench --unstable bench/clear_timeout.ts",
args: "bench bench/clear_timeout.ts",
exit_code: 0,
output: "bench/clear_timeout.out",
});
itest!(finally_timeout {
args: "bench --unstable bench/finally_timeout.ts",
args: "bench bench/finally_timeout.ts",
exit_code: 1,
output: "bench/finally_timeout.out",
});
itest!(group_baseline {
args: "bench --unstable bench/group_baseline.ts",
args: "bench bench/group_baseline.ts",
exit_code: 0,
output: "bench/group_baseline.out",
});
itest!(unresolved_promise {
args: "bench --unstable bench/unresolved_promise.ts",
args: "bench bench/unresolved_promise.ts",
exit_code: 1,
output: "bench/unresolved_promise.out",
});
itest!(unhandled_rejection {
args: "bench --unstable bench/unhandled_rejection.ts",
args: "bench bench/unhandled_rejection.ts",
exit_code: 1,
output: "bench/unhandled_rejection.out",
});
itest!(filter {
args: "bench --unstable --filter=foo bench/filter",
args: "bench --filter=foo bench/filter",
exit_code: 0,
output: "bench/filter.out",
});
itest!(no_prompt_by_default {
args: "bench --quiet --unstable bench/no_prompt_by_default.ts",
args: "bench --quiet bench/no_prompt_by_default.ts",
exit_code: 1,
output: "bench/no_prompt_by_default.out",
});
itest!(no_prompt_with_denied_perms {
args:
"bench --quiet --unstable --allow-read bench/no_prompt_with_denied_perms.ts",
args: "bench --quiet --allow-read bench/no_prompt_with_denied_perms.ts",
exit_code: 1,
output: "bench/no_prompt_with_denied_perms.out",
});
itest!(check_local_by_default {
args: "bench --quiet --unstable bench/check_local_by_default.ts",
args: "bench --quiet bench/check_local_by_default.ts",
output: "bench/check_local_by_default.out",
http_server: true,
});
itest!(check_local_by_default2 {
args: "bench --quiet --unstable bench/check_local_by_default2.ts",
args: "bench --quiet bench/check_local_by_default2.ts",
output: "bench/check_local_by_default2.out",
http_server: true,
exit_code: 1,
@ -173,7 +166,6 @@ fn recursive_permissions_pledge() {
let output = util::deno_cmd()
.current_dir(util::testdata_path())
.arg("bench")
.arg("--unstable")
.arg("bench/recursive_permissions_pledge.js")
.stderr(std::process::Stdio::piped())
.spawn()
@ -194,7 +186,7 @@ fn file_protocol() {
.to_string();
(util::CheckOutputIntegrationTest {
args_vec: vec!["bench", "--unstable", &file_url],
args_vec: vec!["bench", &file_url],
exit_code: 0,
output: "bench/file_protocol.out",
..Default::default()

View file

@ -1 +0,0 @@
Deno.bench("bench0", () => {});

View file

@ -1 +0,0 @@
Unstable API 'Deno.bench'. The --unstable flag must be provided.

View file

@ -356,11 +356,7 @@ async fn bench_specifier(
&ps,
specifier.clone(),
permissions,
vec![ops::bench::init(
channel.clone(),
filter,
ps.options.unstable(),
)],
vec![ops::bench::init(channel.clone(), filter)],
Default::default(),
)
.await?;

View file

@ -737,7 +737,6 @@
return;
}
ops.op_bench_check_unstable();
let benchDesc;
const defaults = {
ignore: false,