Improve run_and_collect_output (#4272)

* Fix flaky test test_permissions_net_listen_allow_localhost

* Improve error reporting for run_and_collect_output
This commit is contained in:
Ryan Dahl 2020-03-07 05:27:16 -05:00 committed by GitHub
parent 61f12ea775
commit ec1946a95e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23

View file

@ -468,7 +468,8 @@ fn bundle_js() {
#[test]
fn repl_test_console_log() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["console.log('hello')", "'world'"]),
None,
@ -476,21 +477,25 @@ fn repl_test_console_log() {
);
assert_eq!(out, "hello\nundefined\nworld\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_eof() {
let (out, err, code) =
util::run_and_collect_output("repl", Some(vec!["1 + 2"]), None, false);
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["1 + 2"]),
None,
false,
);
assert_eq!(out, "3\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_exit_command() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["exit", "'ignored'"]),
None,
@ -498,13 +503,12 @@ fn repl_test_exit_command() {
);
assert!(out.is_empty());
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_help_command() {
let (out, err, code) =
util::run_and_collect_output("repl", Some(vec!["help"]), None, false);
let (out, err) =
util::run_and_collect_output(true, "repl", Some(vec!["help"]), None, false);
assert_eq!(
out,
vec![
@ -517,12 +521,12 @@ fn repl_test_help_command() {
.join("\n")
);
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_function() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["Deno.writeFileSync"]),
None,
@ -530,12 +534,12 @@ fn repl_test_function() {
);
assert_eq!(out, "[Function: writeFileSync]\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_multiline() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["(\n1 + 2\n)"]),
None,
@ -543,21 +547,25 @@ fn repl_test_multiline() {
);
assert_eq!(out, "3\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_eval_unterminated() {
let (out, err, code) =
util::run_and_collect_output("repl", Some(vec!["eval('{')"]), None, false);
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["eval('{')"]),
None,
false,
);
assert!(out.is_empty());
assert!(err.contains("Unexpected end of input"));
assert_eq!(code, 0);
}
#[test]
fn repl_test_reference_error() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["not_a_variable"]),
None,
@ -565,12 +573,12 @@ fn repl_test_reference_error() {
);
assert!(out.is_empty());
assert!(err.contains("not_a_variable is not defined"));
assert_eq!(code, 0);
}
#[test]
fn repl_test_syntax_error() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["syntax error"]),
None,
@ -578,21 +586,25 @@ fn repl_test_syntax_error() {
);
assert!(out.is_empty());
assert!(err.contains("Unexpected identifier"));
assert_eq!(code, 0);
}
#[test]
fn repl_test_type_error() {
let (out, err, code) =
util::run_and_collect_output("repl", Some(vec!["console()"]), None, false);
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["console()"]),
None,
false,
);
assert!(out.is_empty());
assert!(err.contains("console is not a function"));
assert_eq!(code, 0);
}
#[test]
fn repl_test_variable() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["var a = 123;", "a"]),
None,
@ -600,12 +612,12 @@ fn repl_test_variable() {
);
assert_eq!(out, "undefined\n123\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_lexical_scoped_variable() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["let a = 123;", "a"]),
None,
@ -613,7 +625,6 @@ fn repl_test_lexical_scoped_variable() {
);
assert_eq!(out, "undefined\n123\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
@ -622,7 +633,8 @@ fn repl_test_missing_deno_dir() {
const DENO_DIR: &str = "nonexistent";
let test_deno_dir =
util::root_path().join("cli").join("tests").join(DENO_DIR);
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["1"]),
Some(vec![("DENO_DIR".to_owned(), DENO_DIR.to_owned())]),
@ -632,21 +644,25 @@ fn repl_test_missing_deno_dir() {
remove_dir_all(&test_deno_dir).unwrap();
assert_eq!(out, "1\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_save_last_eval() {
let (out, err, code) =
util::run_and_collect_output("repl", Some(vec!["1", "_"]), None, false);
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["1", "_"]),
None,
false,
);
assert_eq!(out, "1\n1\n");
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_save_last_thrown() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["throw 1", "_error"]),
None,
@ -654,12 +670,12 @@ fn repl_test_save_last_thrown() {
);
assert_eq!(out, "1\n");
assert_eq!(err, "Thrown: 1\n");
assert_eq!(code, 0);
}
#[test]
fn repl_test_assign_underscore() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["_ = 1", "2", "_"]),
None,
@ -670,12 +686,12 @@ fn repl_test_assign_underscore() {
"Last evaluation result is no longer saved to _.\n1\n2\n1\n"
);
assert!(err.is_empty());
assert_eq!(code, 0);
}
#[test]
fn repl_test_assign_underscore_error() {
let (out, err, code) = util::run_and_collect_output(
let (out, err) = util::run_and_collect_output(
true,
"repl",
Some(vec!["_error = 1", "throw 2", "_error"]),
None,
@ -686,7 +702,6 @@ fn repl_test_assign_underscore_error() {
"Last thrown error is no longer saved to _error.\n1\n1\n"
);
assert_eq!(err, "Thrown: 2\n");
assert_eq!(code, 0);
}
#[test]
@ -1556,13 +1571,13 @@ fn cafile_bundle_remote_exports() {
#[test]
fn test_permissions_with_allow() {
for permission in &util::PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!("run --allow-{0} permission_test.ts {0}Required", permission),
None,
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1570,13 +1585,13 @@ fn test_permissions_with_allow() {
#[test]
fn test_permissions_without_allow() {
for permission in &util::PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
&format!("run permission_test.ts {0}Required", permission),
None,
None,
false,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1585,7 +1600,8 @@ fn test_permissions_without_allow() {
fn test_permissions_rw_inside_project_dir() {
const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"];
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!(
"run --allow-{0}={1} complex_permissions_test.ts {0} {2} {2}",
permission,
@ -1596,7 +1612,6 @@ fn test_permissions_rw_inside_project_dir() {
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1605,7 +1620,8 @@ fn test_permissions_rw_inside_project_dir() {
fn test_permissions_rw_outside_test_dir() {
const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"];
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
&format!(
"run --allow-{0}={1} complex_permissions_test.ts {0} {2}",
permission,
@ -1625,7 +1641,6 @@ fn test_permissions_rw_outside_test_dir() {
None,
false,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1634,7 +1649,8 @@ fn test_permissions_rw_outside_test_dir() {
fn test_permissions_rw_inside_test_dir() {
const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"];
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!(
"run --allow-{0}={1} complex_permissions_test.ts {0} {2}",
permission,
@ -1650,7 +1666,6 @@ fn test_permissions_rw_inside_test_dir() {
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1670,7 +1685,8 @@ fn test_permissions_rw_outside_test_and_js_dir() {
.into_string()
.unwrap();
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
&format!(
"run --allow-{0}={1},{2} complex_permissions_test.ts {0} {3}",
permission,
@ -1686,7 +1702,6 @@ fn test_permissions_rw_outside_test_and_js_dir() {
None,
false,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1706,7 +1721,8 @@ fn test_permissions_rw_inside_test_and_js_dir() {
.into_string()
.unwrap();
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!(
"run --allow-{0}={1},{2} complex_permissions_test.ts {0} {3}",
permission, test_dir, js_dir, "complex_permissions_test.ts"
@ -1715,7 +1731,6 @@ fn test_permissions_rw_inside_test_and_js_dir() {
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1724,7 +1739,8 @@ fn test_permissions_rw_inside_test_and_js_dir() {
fn test_permissions_rw_relative() {
const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"];
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!(
"run --allow-{0}=. complex_permissions_test.ts {0} complex_permissions_test.ts",
permission
@ -1733,7 +1749,6 @@ fn test_permissions_rw_relative() {
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
@ -1742,7 +1757,8 @@ fn test_permissions_rw_relative() {
fn test_permissions_rw_no_prefix() {
const PERMISSION_VARIANTS: [&str; 2] = ["read", "write"];
for permission in &PERMISSION_VARIANTS {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
&format!(
"run --allow-{0}=tls/../ complex_permissions_test.ts {0} complex_permissions_test.ts",
permission
@ -1751,152 +1767,151 @@ fn test_permissions_rw_no_prefix() {
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
}
#[test]
fn test_permissions_net_fetch_allow_localhost_4545() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=localhost:4545 complex_permissions_test.ts netFetch http://localhost:4545/",
None,
None,true,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_fetch_allow_deno_land() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=deno.land complex_permissions_test.ts netFetch http://localhost:4545/",
None,
None,
true,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_fetch_localhost_4545_fail() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=localhost:4545 complex_permissions_test.ts netFetch http://localhost:4546/",
None,
None,
true,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_fetch_localhost() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=localhost complex_permissions_test.ts netFetch http://localhost:4545/ http://localhost:4546/ http://localhost:4547/",
None,
None,
true,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_connect_allow_localhost_ip_4555() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=127.0.0.1:4545 complex_permissions_test.ts netConnect 127.0.0.1:4545",
None,
None,
true,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_connect_allow_deno_land() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=deno.land complex_permissions_test.ts netConnect 127.0.0.1:4546",
None,
None,
true,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_connect_allow_localhost_ip_4545_fail() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=127.0.0.1:4545 complex_permissions_test.ts netConnect 127.0.0.1:4546",
None,
None,
true,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_connect_allow_localhost_ip() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=127.0.0.1 complex_permissions_test.ts netConnect 127.0.0.1:4545 127.0.0.1:4546 127.0.0.1:4547",
None,
None,
true,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_listen_allow_localhost_4555() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=localhost:4558 complex_permissions_test.ts netListen localhost:4558",
None,
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_listen_allow_deno_land() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=deno.land complex_permissions_test.ts netListen localhost:4545",
None,
None,
false,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_listen_allow_localhost_4555_fail() {
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
false,
"run --allow-net=localhost:4555 complex_permissions_test.ts netListen localhost:4556",
None,
None,
false,
);
assert_eq!(code, 1);
assert!(err.contains(util::PERMISSION_DENIED_PATTERN));
}
#[test]
fn test_permissions_net_listen_allow_localhost() {
// Port 4600 is chosen to not colide with those used by tools/http_server.py
let (_, err, code) = util::run_and_collect_output(
let (_, err) = util::run_and_collect_output(
true,
"run --allow-net=localhost complex_permissions_test.ts netListen localhost:4600",
None,
None,
false,
);
assert_eq!(code, 0);
assert!(!err.contains(util::PERMISSION_DENIED_PATTERN));
}
@ -1920,11 +1935,12 @@ mod util {
}
pub fn run_and_collect_output(
expect_success: bool,
args: &str,
input: Option<Vec<&str>>,
envs: Option<Vec<(String, String)>>,
need_http_server: bool,
) -> (String, String, i32) {
) -> (String, String) {
let root = root_path();
let tests_dir = root.join("cli").join("tests");
let mut deno_process_builder = deno_cmd();
@ -1957,11 +1973,14 @@ mod util {
status,
} = deno.wait_with_output().expect("failed to wait on child");
drop(http_guard);
(
String::from_utf8(stdout).unwrap(),
String::from_utf8(stderr).unwrap(),
status.code().unwrap(),
)
let stdout = String::from_utf8(stdout).unwrap();
let stderr = String::from_utf8(stderr).unwrap();
if expect_success != status.success() {
eprintln!("stdout: <<<{}>>>", stdout);
eprintln!("stderr: <<<{}>>>", stderr);
panic!("Unexpected exit code: {:?}", status.code());
}
(stdout, stderr)
}
pub fn deno_cmd() -> Command {