test(cli): harden test runner tests (#11166)

This commit is contained in:
Casper Beyer 2021-06-29 21:40:16 +08:00 committed by GitHub
parent 96d0582900
commit a0c0daac24
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
34 changed files with 447 additions and 170 deletions

View file

@ -1,4 +0,0 @@
[WILDCARD]error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
const _a: string = 1;
~~
at [WILDCARD]foo_test.ts[WILDCARD]

View file

@ -7,7 +7,7 @@ use test_util as util;
fn no_color() {
let (out, _) = util::run_and_collect_output(
false,
"test test/deno_test_no_color.ts",
"test test/no_color.ts",
None,
Some(vec![("NO_COLOR".to_owned(), "true".to_owned())]),
false,
@ -19,10 +19,22 @@ fn no_color() {
assert!(out.contains("test result: FAILED. 1 passed; 1 failed; 1 ignored; 0 measured; 0 filtered out"));
}
itest!(all {
args: "test test/test_runner_test.ts",
itest!(pass {
args: "test test/pass.ts",
exit_code: 0,
output: "test/pass.out",
});
itest!(ignore {
args: "test test/ignore.ts",
exit_code: 0,
output: "test/ignore.out",
});
itest!(fail {
args: "test test/fail.ts",
exit_code: 1,
output: "test/deno_test.out",
output: "test/fail.out",
});
itest!(doc {
@ -31,6 +43,36 @@ itest!(doc {
output: "test/doc.out",
});
itest!(quiet {
args: "test --quiet test/quiet.ts",
exit_code: 0,
output: "test/quiet.out",
});
itest!(fail_fast {
args: "test --fail-fast test/fail_fast.ts",
exit_code: 1,
output: "test/fail_fast.out",
});
itest!(only {
args: "test test/only.ts",
exit_code: 1,
output: "test/only.out",
});
itest!(no_check {
args: "test --no-check test/no_check.ts",
exit_code: 1,
output: "test/no_check.out",
});
itest!(no_run {
args: "test --unstable --no-run test/no_run.ts",
output: "test/no_run.out",
exit_code: 1,
});
itest!(allow_all {
args: "test --unstable --allow-all test/allow_all.ts",
exit_code: 0,
@ -43,34 +85,22 @@ itest!(allow_none {
output: "test/allow_none.out",
});
itest!(fail_fast {
args: "test --fail-fast test/test_runner_test.ts",
itest!(exit_sanitizer {
args: "test test/exit_sanitizer.ts",
output: "test/exit_sanitizer.out",
exit_code: 1,
output: "test/deno_test_fail_fast.out",
});
itest!(only {
args: "test test/deno_test_only.ts",
itest!(finally_timeout {
args: "test test/finally_timeout.ts",
exit_code: 1,
output: "test/deno_test_only.ts.out",
});
itest!(no_check {
args: "test --no-check test/test_runner_test.ts",
exit_code: 1,
output: "test/deno_test.out",
});
itest!(finally_cleartimeout {
args: "test test/test_finally_cleartimeout.ts",
exit_code: 1,
output: "test/test_finally_cleartimeout.out",
output: "test/finally_timeout.out",
});
itest!(unresolved_promise {
args: "test test/test_unresolved_promise.js",
args: "test test/unresolved_promise.ts",
exit_code: 1,
output: "test/deno_test_unresolved_promise.out",
output: "test/unresolved_promise.out",
});
itest!(unhandled_rejection {
@ -78,21 +108,3 @@ itest!(unhandled_rejection {
exit_code: 1,
output: "test/unhandled_rejection.out",
});
itest!(exit_sanitizer {
args: "test test/exit_sanitizer_test.ts",
output: "test/exit_sanitizer_test.out",
exit_code: 1,
});
itest!(quiet {
args: "test --quiet test/quiet_test.ts",
exit_code: 0,
output: "test/quiet_test.out",
});
itest!(_067_test_no_run_type_error {
args: "test --unstable --no-run test_type_error",
output: "067_test_no_run_type_error.out",
exit_code: 1,
});

View file

@ -1,26 +0,0 @@
[WILDCARD]
running 4 tests from [WILDCARD]
test fail1 ... FAILED [WILDCARD]
test fail2 ... FAILED [WILDCARD]
test success1 ... ok [WILDCARD]
test fail3 ... FAILED [WILDCARD]
failures:
fail1
AssertionError: fail1 assertion
[WILDCARD]
fail2
AssertionError: fail2 assertion
[WILDCARD]
fail3
AssertionError: fail3 assertion
[WILDCARD]
failures:
[WILDCARD]
test result: FAILED. 1 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]

View file

@ -1,15 +0,0 @@
[WILDCARD]
running 4 tests from [WILDCARD]
test fail1 ... FAILED [WILDCARD]
failures:
fail1
AssertionError: fail1 assertion
[WILDCARD]
failures:
[WILDCARD]
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]

View file

@ -0,0 +1,55 @@
Check [WILDCARD]/test/exit_sanitizer.ts
running 3 tests from [WILDCARD]/test/exit_sanitizer.ts
test exit(0) ... FAILED ([WILDCARD])
test exit(1) ... FAILED ([WILDCARD])
test exit(2) ... FAILED ([WILDCARD])
failures:
exit(0)
AssertionError: Test case attempted to exit with exit code: 0
at assert (deno:runtime/js/06_util.js:33:13)
at deno:runtime/js/40_testing.js:78:9
at Object.exit (deno:runtime/js/30_os.js:48:7)
at [WILDCARD]/test/exit_sanitizer.ts:2:8
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at [WILDCARD]/$deno$test.js:1:27
exit(1)
AssertionError: Test case attempted to exit with exit code: 1
at assert (deno:runtime/js/06_util.js:33:13)
at deno:runtime/js/40_testing.js:78:9
at Object.exit (deno:runtime/js/30_os.js:48:7)
at [WILDCARD]/test/exit_sanitizer.ts:6:8
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
exit(2)
AssertionError: Test case attempted to exit with exit code: 2
at assert (deno:runtime/js/06_util.js:33:13)
at deno:runtime/js/40_testing.js:78:9
at Object.exit (deno:runtime/js/30_os.js:48:7)
at [WILDCARD]/test/exit_sanitizer.ts:10:8
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
failures:
exit(0)
exit(1)
exit(2)
test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

View file

@ -1,28 +0,0 @@
Check [WILDCARD]/exit_sanitizer_test.ts
running 3 tests from [WILDCARD]
test exit(0) ... FAILED ([WILDCARD])
test exit(1) ... FAILED ([WILDCARD])
test exit(2) ... FAILED ([WILDCARD])
failures:
exit(0)
AssertionError: Test case attempted to exit with exit code: 0
[WILDCARD]
exit(1)
AssertionError: Test case attempted to exit with exit code: 1
[WILDCARD]
exit(2)
AssertionError: Test case attempted to exit with exit code: 2
[WILDCARD]
failures:
exit(0)
exit(1)
exit(2)
test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

130
cli/tests/test/fail.out Normal file
View file

@ -0,0 +1,130 @@
Check [WILDCARD]/test/fail.ts
running 10 tests from [WILDCARD]/test/fail.ts
test test 0 ... FAILED ([WILDCARD])
test test 1 ... FAILED ([WILDCARD])
test test 2 ... FAILED ([WILDCARD])
test test 3 ... FAILED ([WILDCARD])
test test 4 ... FAILED ([WILDCARD])
test test 5 ... FAILED ([WILDCARD])
test test 6 ... FAILED ([WILDCARD])
test test 7 ... FAILED ([WILDCARD])
test test 8 ... FAILED ([WILDCARD])
test test 9 ... FAILED ([WILDCARD])
failures:
test 0
Error
at [WILDCARD]/test/fail.ts:2:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at [WILDCARD]/$deno$test.js:1:27
test 1
Error
at [WILDCARD]/test/fail.ts:5:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 2
Error
at [WILDCARD]/test/fail.ts:8:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 3
Error
at [WILDCARD]/test/fail.ts:11:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 4
Error
at [WILDCARD]/test/fail.ts:14:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 5
Error
at [WILDCARD]/test/fail.ts:17:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 6
Error
at [WILDCARD]/test/fail.ts:20:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 7
Error
at [WILDCARD]/test/fail.ts:23:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 8
Error
at [WILDCARD]/test/fail.ts:26:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
test 9
Error
at [WILDCARD]/test/fail.ts:29:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at async [WILDCARD]/$deno$test.js:1:1
failures:
test 0
test 1
test 2
test 3
test 4
test 5
test 6
test 7
test 8
test 9
test result: FAILED. 0 passed; 10 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

30
cli/tests/test/fail.ts Normal file
View file

@ -0,0 +1,30 @@
Deno.test("test 0", () => {
throw new Error();
});
Deno.test("test 1", () => {
throw new Error();
});
Deno.test("test 2", () => {
throw new Error();
});
Deno.test("test 3", () => {
throw new Error();
});
Deno.test("test 4", () => {
throw new Error();
});
Deno.test("test 5", () => {
throw new Error();
});
Deno.test("test 6", () => {
throw new Error();
});
Deno.test("test 7", () => {
throw new Error();
});
Deno.test("test 8", () => {
throw new Error();
});
Deno.test("test 9", () => {
throw new Error();
});

View file

@ -0,0 +1,22 @@
Check [WILDCARD]/test/fail_fast.ts
running 10 tests from [WILDCARD]/test/fail_fast.ts
test test 1 ... FAILED ([WILDCARD])
failures:
test 1
Error
at [WILDCARD]/test/fail_fast.ts:2:9
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at [WILDCARD]/$deno$test.js:1:27
failures:
test 1
test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

View file

@ -0,0 +1,30 @@
Deno.test("test 1", () => {
throw new Error();
});
Deno.test("test 2", () => {
throw new Error();
});
Deno.test("test 3", () => {
throw new Error();
});
Deno.test("test 4", () => {
throw new Error();
});
Deno.test("test 5", () => {
throw new Error();
});
Deno.test("test 6", () => {
throw new Error();
});
Deno.test("test 7", () => {
throw new Error();
});
Deno.test("test 8", () => {
throw new Error();
});
Deno.test("test 9", () => {
throw new Error();
});
Deno.test("test 0", () => {
throw new Error();
});

View file

@ -0,0 +1,23 @@
Check [WILDCARD]/test/finally_timeout.ts
running 2 tests from [WILDCARD]/test/finally_timeout.ts
test error ... FAILED ([WILDCARD])
test success ... ok ([WILDCARD])
failures:
error
Error: fail
at [WILDCARD]/test/finally_timeout.ts:4:11
at asyncOpSanitizer (deno:runtime/js/40_testing.js:21:15)
at resourceSanitizer (deno:runtime/js/40_testing.js:58:13)
at exitSanitizer (deno:runtime/js/40_testing.js:85:15)
at runTest (deno:runtime/js/40_testing.js:199:13)
at Object.runTests (deno:runtime/js/40_testing.js:244:13)
at [WILDCARD]/$deno$test.js:1:27
failures:
error
test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

15
cli/tests/test/ignore.out Normal file
View file

@ -0,0 +1,15 @@
Check [WILDCARD]/test/ignore.ts
running 10 tests from [WILDCARD]/test/ignore.ts
test test 0 ... ignored ([WILDCARD])
test test 1 ... ignored ([WILDCARD])
test test 2 ... ignored ([WILDCARD])
test test 3 ... ignored ([WILDCARD])
test test 4 ... ignored ([WILDCARD])
test test 5 ... ignored ([WILDCARD])
test test 6 ... ignored ([WILDCARD])
test test 7 ... ignored ([WILDCARD])
test test 8 ... ignored ([WILDCARD])
test test 9 ... ignored ([WILDCARD])
test result: ok. 0 passed; 0 failed; 10 ignored; 0 measured; 0 filtered out ([WILDCARD])

9
cli/tests/test/ignore.ts Normal file
View file

@ -0,0 +1,9 @@
for (let i = 0; i < 10; i++) {
Deno.test({
name: `test ${i}`,
ignore: true,
fn() {
throw new Error("unreachable");
},
});
}

View file

@ -0,0 +1,8 @@
test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
error: Uncaught TypeError: Cannot read property 'fn' of undefined
Deno.test();
^
at Object.test (deno:runtime/js/40_testing.js:121:14)
at [WILDCARD]/test/no_check.ts:1:6

View file

@ -0,0 +1 @@
Deno.test();

View file

@ -0,0 +1,5 @@
Check [WILDCARD]/test/no_run.ts
error: TS2322 [ERROR]: Type 'number' is not assignable to type 'string'.
const _value: string = 1;
~~~~~~
at [WILDCARD]/test/no_run.ts:1:7

1
cli/tests/test/no_run.ts Normal file
View file

@ -0,0 +1 @@
const _value: string = 1;

View file

@ -1,6 +1,6 @@
[WILDCARD]
running 1 test from [WILDCARD]
test def ... ok ([WILDCARD])
Check [WILDCARD]/test/only.ts
running 1 test from [WILDCARD]/test/only.ts
test only ... ok ([WILDCARD])
test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 2 filtered out ([WILDCARD])

View file

@ -1,15 +1,15 @@
Deno.test({
name: "abc",
name: "before",
fn() {},
});
Deno.test({
only: true,
name: "def",
name: "only",
fn() {},
});
Deno.test({
name: "ghi",
name: "after",
fn() {},
});

15
cli/tests/test/pass.out Normal file
View file

@ -0,0 +1,15 @@
Check [WILDCARD]/test/pass.ts
running 10 tests from [WILDCARD]/test/pass.ts
test test 0 ... ok ([WILDCARD])
test test 1 ... ok ([WILDCARD])
test test 2 ... ok ([WILDCARD])
test test 3 ... ok ([WILDCARD])
test test 4 ... ok ([WILDCARD])
test test 5 ... ok ([WILDCARD])
test test 6 ... ok ([WILDCARD])
test test 7 ... ok ([WILDCARD])
test test 8 ... ok ([WILDCARD])
test test 9 ... ok ([WILDCARD])
test result: ok. 10 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

10
cli/tests/test/pass.ts Normal file
View file

@ -0,0 +1,10 @@
Deno.test("test 0", () => {});
Deno.test("test 1", () => {});
Deno.test("test 2", () => {});
Deno.test("test 3", () => {});
Deno.test("test 4", () => {});
Deno.test("test 5", () => {});
Deno.test("test 6", () => {});
Deno.test("test 7", () => {});
Deno.test("test 8", () => {});
Deno.test("test 9", () => {});

8
cli/tests/test/quiet.out Normal file
View file

@ -0,0 +1,8 @@
running 4 tests from [WILDCARD]/test/quiet.ts
test console.log ... ok ([WILDCARD])
test console.error ... ok ([WILDCARD])
test console.info ... ok ([WILDCARD])
test console.warn ... ok ([WILDCARD])
test result: ok. 4 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

15
cli/tests/test/quiet.ts Normal file
View file

@ -0,0 +1,15 @@
Deno.test("console.log", function () {
console.log("log");
});
Deno.test("console.error", function () {
console.error("error");
});
Deno.test("console.info", function () {
console.info("info");
});
Deno.test("console.warn", function () {
console.info("warn");
});

View file

@ -1,4 +0,0 @@
running 1 test from [WILDCARD]
test log ... ok [WILDCARD]
test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]

View file

@ -1,3 +0,0 @@
Deno.test("log", function () {
console.log("log");
});

View file

@ -1,17 +0,0 @@
Check [WILDCARD]/test_finally_cleartimeout.ts
running 2 tests from [WILDCARD]
test error ... FAILED ([WILDCARD])
test success ... ok ([WILDCARD])
failures:
error
Error: fail
[WILDCARD]
failures:
error
test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])

View file

@ -1,19 +0,0 @@
// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
import { assert } from "../../../test_util/std/testing/asserts.ts";
Deno.test("fail1", function () {
assert(false, "fail1 assertion");
});
Deno.test("fail2", function () {
assert(false, "fail2 assertion");
});
Deno.test("success1", function () {
assert(true);
});
Deno.test("fail3", function () {
assert(false, "fail3 assertion");
});

View file

@ -1,6 +1,10 @@
Check [WILDCARD]
Check [WILDCARD]/test/unhandled_rejection.ts
test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
error: Uncaught (in promise) Error: rejection
[WILDCARD]
reject(new Error("rejection"));
^
at [WILDCARD]/test/unhandled_rejection.ts:2:10
at new Promise (<anonymous>)
at [WILDCARD]/test/unhandled_rejection.ts:1:1

View file

@ -1,5 +1,6 @@
running 2 tests from [WILDCARD]
Check [WILDCARD]/test/unresolved_promise.ts
running 2 tests from [WILDCARD]/test/unresolved_promise.ts
test unresolved promise ...
test result: FAILED. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
test result: FAILED. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
error: Module evaluation is still pending but there are no pending ops or dynamic imports. This situation is often caused by unresolved promise.

View file

@ -1 +0,0 @@
const _a: string = 1;