From 3e03865d89e3abf0755e6d3b8305632a5319fdfe Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Sun, 21 May 2023 03:43:54 +0200 Subject: [PATCH] feat(unstable): add more options to Deno.createHttpClient (#17385) --- cli/file_fetcher.rs | 93 ++++++++++----------- cli/http_util.rs | 28 +++---- cli/tests/unit/fetch_test.ts | 34 ++++++++ cli/tsc/dts/lib.deno.unstable.d.ts | 9 ++ ext/fetch/lib.rs | 130 +++++++++++++++++++++++------ test_util/src/lib.rs | 42 ++++++++-- 6 files changed, 242 insertions(+), 94 deletions(-) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index fd8c0f7939..71d284ef6d 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -744,6 +744,7 @@ mod tests { use deno_core::resolve_url; use deno_core::url::Url; use deno_runtime::deno_fetch::create_http_client; + use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_web::Blob; use deno_runtime::deno_web::InMemoryBlobPart; use std::fs::read; @@ -1746,7 +1747,7 @@ mod tests { fn create_test_client() -> HttpClient { HttpClient::from_client( - create_http_client("test_client", None, vec![], None, None, None) + create_http_client("test_client", CreateHttpClientOptions::default()) .unwrap(), ) } @@ -1943,17 +1944,16 @@ mod tests { let client = HttpClient::from_client( create_http_client( version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), - ) - .unwrap()], - None, - None, - None, + CreateHttpClientOptions { + ca_certs: vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + ..Default::default() + }, ) .unwrap(), ); @@ -1986,11 +1986,7 @@ mod tests { let client = HttpClient::from_client( create_http_client( version::get_user_agent(), - None, // This will load mozilla certs by default - vec![], - None, - None, - None, + CreateHttpClientOptions::default(), ) .unwrap(), ); @@ -2068,17 +2064,16 @@ mod tests { let client = HttpClient::from_client( create_http_client( version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), - ) - .unwrap()], - None, - None, - None, + CreateHttpClientOptions { + ca_certs: vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + ..Default::default() + }, ) .unwrap(), ); @@ -2113,17 +2108,16 @@ mod tests { let client = HttpClient::from_client( create_http_client( version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), - ) - .unwrap()], - None, - None, - None, + CreateHttpClientOptions { + ca_certs: vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + ..Default::default() + }, ) .unwrap(), ); @@ -2175,17 +2169,16 @@ mod tests { let client = HttpClient::from_client( create_http_client( version::get_user_agent(), - None, - vec![read( - test_util::testdata_path() - .join("tls/RootCA.pem") - .to_str() - .unwrap(), - ) - .unwrap()], - None, - None, - None, + CreateHttpClientOptions { + ca_certs: vec![read( + test_util::testdata_path() + .join("tls/RootCA.pem") + .to_str() + .unwrap(), + ) + .unwrap()], + ..Default::default() + }, ) .unwrap(), ); diff --git a/cli/http_util.rs b/cli/http_util.rs index 7c17e8e1e5..e90e0ee96d 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -15,6 +15,7 @@ use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::reqwest; use deno_runtime::deno_fetch::reqwest::header::LOCATION; use deno_runtime::deno_fetch::reqwest::Response; +use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_tls::RootCertStoreProvider; use std::collections::HashMap; use std::sync::Arc; @@ -219,18 +220,15 @@ impl CacheSemantics { } pub struct HttpClient { + options: CreateHttpClientOptions, root_cert_store_provider: Option>, - unsafely_ignore_certificate_errors: Option>, cell: once_cell::sync::OnceCell, } impl std::fmt::Debug for HttpClient { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("HttpClient") - .field( - "unsafely_ignore_certificate_errors", - &self.unsafely_ignore_certificate_errors, - ) + .field("options", &self.options) .finish() } } @@ -241,8 +239,11 @@ impl HttpClient { unsafely_ignore_certificate_errors: Option>, ) -> Self { Self { + options: CreateHttpClientOptions { + unsafely_ignore_certificate_errors, + ..Default::default() + }, root_cert_store_provider, - unsafely_ignore_certificate_errors, cell: Default::default(), } } @@ -250,8 +251,8 @@ impl HttpClient { #[cfg(test)] pub fn from_client(client: reqwest::Client) -> Self { let result = Self { + options: Default::default(), root_cert_store_provider: Default::default(), - unsafely_ignore_certificate_errors: Default::default(), cell: Default::default(), }; result.cell.set(client).unwrap(); @@ -262,14 +263,13 @@ impl HttpClient { self.cell.get_or_try_init(|| { create_http_client( get_user_agent(), - match &self.root_cert_store_provider { - Some(provider) => Some(provider.get_or_try_init()?.clone()), - None => None, + CreateHttpClientOptions { + root_cert_store: match &self.root_cert_store_provider { + Some(provider) => Some(provider.get_or_try_init()?.clone()), + None => None, + }, + ..self.options.clone() }, - vec![], - None, - self.unsafely_ignore_certificate_errors.clone(), - None, ) }) } diff --git a/cli/tests/unit/fetch_test.ts b/cli/tests/unit/fetch_test.ts index 7de04013ea..d867955782 100644 --- a/cli/tests/unit/fetch_test.ts +++ b/cli/tests/unit/fetch_test.ts @@ -1531,6 +1531,40 @@ Deno.test( }, ); +Deno.test( + { + permissions: { net: true, read: true }, + // Doesn't pass on linux CI for unknown reasons (works fine locally on linux) + ignore: Deno.build.os !== "darwin", + }, + async function fetchForceHttp1OnHttp2Server() { + const client = Deno.createHttpClient({ http2: false, http1: true }); + await assertRejects( + () => fetch("http://localhost:5549/http_version", { client }), + TypeError, + "invalid HTTP version parsed", + ); + client.close(); + }, +); + +Deno.test( + { + permissions: { net: true, read: true }, + // Doesn't pass on linux CI for unknown reasons (works fine locally on linux) + ignore: Deno.build.os !== "darwin", + }, + async function fetchForceHttp2OnHttp1Server() { + const client = Deno.createHttpClient({ http2: true, http1: false }); + await assertRejects( + () => fetch("http://localhost:5548/http_version", { client }), + TypeError, + "stream closed because of a broken pipe", + ); + client.close(); + }, +); + Deno.test( { permissions: { net: true, read: true } }, async function fetchPrefersHttp2() { diff --git a/cli/tsc/dts/lib.deno.unstable.d.ts b/cli/tsc/dts/lib.deno.unstable.d.ts index 70d7ef7c4f..8681cbd9b1 100644 --- a/cli/tsc/dts/lib.deno.unstable.d.ts +++ b/cli/tsc/dts/lib.deno.unstable.d.ts @@ -821,6 +821,15 @@ declare namespace Deno { certChain?: string; /** PEM formatted (RSA or PKCS8) private key of client certificate. */ privateKey?: string; + /** Sets the maximum numer of idle connections per host allowed in the pool. */ + poolMaxIdlePerHost?: number; + /** Set an optional timeout for idle sockets being kept-alive. + * Set to false to disable the timeout. */ + poolIdleTimeout?: number | false; + /** Whether HTTP/1.1 is allowed or not. */ + http1?: boolean; + /** Whether HTTP/2 is allowed or not. */ + http2?: boolean; } /** **UNSTABLE**: New API, yet to be vetted. diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index e41d85ea4a..9fdf80eec1 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -200,11 +200,19 @@ pub fn get_or_create_client_from_state( let options = state.borrow::(); let client = create_http_client( &options.user_agent, - options.root_cert_store()?, - vec![], - options.proxy.clone(), - options.unsafely_ignore_certificate_errors.clone(), - options.client_cert_chain_and_key.clone(), + CreateHttpClientOptions { + root_cert_store: options.root_cert_store()?, + ca_certs: vec![], + proxy: options.proxy.clone(), + unsafely_ignore_certificate_errors: options + .unsafely_ignore_certificate_errors + .clone(), + client_cert_chain_and_key: options.client_cert_chain_and_key.clone(), + pool_max_idle_per_host: None, + pool_idle_timeout: None, + http1: true, + http2: true, + }, )?; state.put::(client.clone()); Ok(client) @@ -606,19 +614,36 @@ impl HttpClientResource { } } +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub enum PoolIdleTimeout { + State(bool), + Specify(u64), +} + #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] -pub struct CreateHttpClientOptions { +pub struct CreateHttpClientArgs { ca_certs: Vec, proxy: Option, cert_chain: Option, private_key: Option, + pool_max_idle_per_host: Option, + pool_idle_timeout: Option, + #[serde(default = "default_true")] + http1: bool, + #[serde(default = "default_true")] + http2: bool, +} + +fn default_true() -> bool { + true } #[op] pub fn op_fetch_custom_client( state: &mut OpState, - args: CreateHttpClientOptions, + args: CreateHttpClientArgs, ) -> Result where FP: FetchPermissions + 'static, @@ -653,32 +678,71 @@ where let client = create_http_client( &options.user_agent, - options.root_cert_store()?, - ca_certs, - args.proxy, - options.unsafely_ignore_certificate_errors.clone(), - client_cert_chain_and_key, + CreateHttpClientOptions { + root_cert_store: options.root_cert_store()?, + ca_certs, + proxy: args.proxy, + unsafely_ignore_certificate_errors: options + .unsafely_ignore_certificate_errors + .clone(), + client_cert_chain_and_key, + pool_max_idle_per_host: args.pool_max_idle_per_host, + pool_idle_timeout: args.pool_idle_timeout.and_then( + |timeout| match timeout { + PoolIdleTimeout::State(true) => None, + PoolIdleTimeout::State(false) => Some(None), + PoolIdleTimeout::Specify(specify) => Some(Some(specify)), + }, + ), + http1: args.http1, + http2: args.http2, + }, )?; let rid = state.resource_table.add(HttpClientResource::new(client)); Ok(rid) } +#[derive(Debug, Clone)] +pub struct CreateHttpClientOptions { + pub root_cert_store: Option, + pub ca_certs: Vec>, + pub proxy: Option, + pub unsafely_ignore_certificate_errors: Option>, + pub client_cert_chain_and_key: Option<(String, String)>, + pub pool_max_idle_per_host: Option, + pub pool_idle_timeout: Option>, + pub http1: bool, + pub http2: bool, +} + +impl Default for CreateHttpClientOptions { + fn default() -> Self { + CreateHttpClientOptions { + root_cert_store: None, + ca_certs: vec![], + proxy: None, + unsafely_ignore_certificate_errors: None, + client_cert_chain_and_key: None, + pool_max_idle_per_host: None, + pool_idle_timeout: None, + http1: true, + http2: true, + } + } +} + /// Create new instance of async reqwest::Client. This client supports /// proxies and doesn't follow redirects. pub fn create_http_client( user_agent: &str, - root_cert_store: Option, - ca_certs: Vec>, - proxy: Option, - unsafely_ignore_certificate_errors: Option>, - client_cert_chain_and_key: Option<(String, String)>, + options: CreateHttpClientOptions, ) -> Result { let mut tls_config = deno_tls::create_client_config( - root_cert_store, - ca_certs, - unsafely_ignore_certificate_errors, - client_cert_chain_and_key, + options.root_cert_store, + options.ca_certs, + options.unsafely_ignore_certificate_errors, + options.client_cert_chain_and_key, )?; tls_config.alpn_protocols = vec!["h2".into(), "http/1.1".into()]; @@ -690,7 +754,7 @@ pub fn create_http_client( .default_headers(headers) .use_preconfigured_tls(tls_config); - if let Some(proxy) = proxy { + if let Some(proxy) = options.proxy { let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?; if let Some(basic_auth) = &proxy.basic_auth { reqwest_proxy = @@ -699,6 +763,24 @@ pub fn create_http_client( builder = builder.proxy(reqwest_proxy); } - // unwrap here because it can only fail when native TLS is used. - Ok(builder.build().unwrap()) + if let Some(pool_max_idle_per_host) = options.pool_max_idle_per_host { + builder = builder.pool_max_idle_per_host(pool_max_idle_per_host); + } + + if let Some(pool_idle_timeout) = options.pool_idle_timeout { + builder = builder.pool_idle_timeout( + pool_idle_timeout.map(std::time::Duration::from_millis), + ); + } + + match (options.http1, options.http2) { + (true, false) => builder = builder.http1_only(), + (false, true) => builder = builder.http2_prior_knowledge(), + (true, true) => {} + (false, false) => { + return Err(type_error("Either `http1` or `http2` needs to be true")) + } + } + + builder.build().map_err(|e| e.into()) } diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index 054298b547..96dc1c325b 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -80,8 +80,10 @@ const TLS_CLIENT_AUTH_PORT: u16 = 4552; const BASIC_AUTH_REDIRECT_PORT: u16 = 4554; const TLS_PORT: u16 = 4557; const HTTPS_PORT: u16 = 5545; -const H1_ONLY_PORT: u16 = 5546; -const H2_ONLY_PORT: u16 = 5547; +const H1_ONLY_TLS_PORT: u16 = 5546; +const H2_ONLY_TLS_PORT: u16 = 5547; +const H1_ONLY_PORT: u16 = 5548; +const H2_ONLY_PORT: u16 = 5549; const HTTPS_CLIENT_AUTH_PORT: u16 = 5552; const WS_PORT: u16 = 4242; const WSS_PORT: u16 = 4243; @@ -1395,8 +1397,9 @@ async fn wrap_main_https_server() { } } -async fn wrap_https_h1_only_server() { - let main_server_https_addr = SocketAddr::from(([127, 0, 0, 1], H1_ONLY_PORT)); +async fn wrap_https_h1_only_tls_server() { + let main_server_https_addr = + SocketAddr::from(([127, 0, 0, 1], H1_ONLY_TLS_PORT)); let cert_file = "tls/localhost.crt"; let key_file = "tls/localhost.key"; let ca_cert_file = "tls/RootCA.pem"; @@ -1440,8 +1443,9 @@ async fn wrap_https_h1_only_server() { } } -async fn wrap_https_h2_only_server() { - let main_server_https_addr = SocketAddr::from(([127, 0, 0, 1], H2_ONLY_PORT)); +async fn wrap_https_h2_only_tls_server() { + let main_server_https_addr = + SocketAddr::from(([127, 0, 0, 1], H2_ONLY_TLS_PORT)); let cert_file = "tls/localhost.crt"; let key_file = "tls/localhost.key"; let ca_cert_file = "tls/RootCA.pem"; @@ -1485,6 +1489,28 @@ async fn wrap_https_h2_only_server() { } } +async fn wrap_https_h1_only_server() { + let main_server_http_addr = SocketAddr::from(([127, 0, 0, 1], H1_ONLY_PORT)); + + let main_server_http_svc = + make_service_fn(|_| async { Ok::<_, Infallible>(service_fn(main_server)) }); + let main_server_http = Server::bind(&main_server_http_addr) + .http1_only(true) + .serve(main_server_http_svc); + let _ = main_server_http.await; +} + +async fn wrap_https_h2_only_server() { + let main_server_http_addr = SocketAddr::from(([127, 0, 0, 1], H2_ONLY_PORT)); + + let main_server_http_svc = + make_service_fn(|_| async { Ok::<_, Infallible>(service_fn(main_server)) }); + let main_server_http = Server::bind(&main_server_http_addr) + .http2_only(true) + .serve(main_server_http_svc); + let _ = main_server_http.await; +} + async fn wrap_client_auth_https_server() { let main_server_https_addr = SocketAddr::from(([127, 0, 0, 1], HTTPS_CLIENT_AUTH_PORT)); @@ -1573,6 +1599,8 @@ pub async fn run_all_servers() { let client_auth_server_https_fut = wrap_client_auth_https_server(); let main_server_fut = wrap_main_server(); let main_server_https_fut = wrap_main_https_server(); + let h1_only_server_tls_fut = wrap_https_h1_only_tls_server(); + let h2_only_server_tls_fut = wrap_https_h2_only_tls_server(); let h1_only_server_fut = wrap_https_h1_only_server(); let h2_only_server_fut = wrap_https_h2_only_server(); @@ -1594,6 +1622,8 @@ pub async fn run_all_servers() { main_server_fut, main_server_https_fut, client_auth_server_https_fut, + h1_only_server_tls_fut, + h2_only_server_tls_fut, h1_only_server_fut, h2_only_server_fut )