feat(fetch): mTLS client certificates for fetch() (#11721)

This commit adds support for specifying client certificates when using fetch, by means of `Deno.createHttpClient`.

cli/dts/lib.deno.unstable.d.ts

@@ -859,6 +859,8 @@ declare namespace Deno {
      */
     caData?: string;
     proxy?: Proxy;
+    certChain?: string;
+    privateKey?: string;
   }
 
   export interface Proxy {

cli/file_fetcher.rs

@@ -237,6 +237,7 @@ impl FileFetcher {
         None,
         None,
         unsafely_ignore_certificate_errors,
+        None,
       )?,
       blob_store,
     })

cli/http_util.rs

@@ -144,8 +144,15 @@ mod tests {
   use std::fs::read;
 
   fn create_test_client(ca_data: Option<Vec<u8>>) -> Client {
-    create_http_client("test_client".to_string(), None, ca_data, None, None)
+    create_http_client(
-      .unwrap()
+      "test_client".to_string(),
+      None,
+      ca_data,
+      None,
+      None,
+      None,
+    )
+    .unwrap()
   }
 
   #[tokio::test]
@@ -340,6 +347,7 @@ mod tests {
       ),
       None,
       None,
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -370,6 +378,7 @@ mod tests {
       None,
       None,
       None,
+      None,
     )
     .unwrap();
 
@@ -402,6 +411,7 @@ mod tests {
       None,
       None,
       None,
+      None,
     )
     .unwrap();
 
@@ -440,6 +450,7 @@ mod tests {
       ),
       None,
       None,
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -480,6 +491,7 @@ mod tests {
       ),
       None,
       None,
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -533,6 +545,7 @@ mod tests {
       ),
       None,
       None,
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {

cli/tests/unit/fetch_test.ts

@@ -1211,3 +1211,83 @@ unitTest(
     assertEquals(res.body, null);
   },
 );
+
+unitTest(
+  { perms: { read: true, net: true } },
+  async function fetchClientCertWrongPrivateKey(): Promise<void> {
+    await assertThrowsAsync(async () => {
+      const client = Deno.createHttpClient({
+        certChain: "bad data",
+        privateKey: await Deno.readTextFile(
+          "cli/tests/testdata/tls/localhost.key",
+        ),
+      });
+      await fetch("https://localhost:5552/fixture.json", {
+        client,
+      });
+    }, Deno.errors.InvalidData);
+  },
+);
+
+unitTest(
+  { perms: { read: true, net: true } },
+  async function fetchClientCertBadPrivateKey(): Promise<void> {
+    await assertThrowsAsync(async () => {
+      const client = Deno.createHttpClient({
+        certChain: await Deno.readTextFile(
+          "cli/tests/testdata/tls/localhost.crt",
+        ),
+        privateKey: "bad data",
+      });
+      await fetch("https://localhost:5552/fixture.json", {
+        client,
+      });
+    }, Deno.errors.InvalidData);
+  },
+);
+
+unitTest(
+  { perms: { read: true, net: true } },
+  async function fetchClientCertNotPrivateKey(): Promise<void> {
+    await assertThrowsAsync(async () => {
+      const client = Deno.createHttpClient({
+        certChain: await Deno.readTextFile(
+          "cli/tests/testdata/tls/localhost.crt",
+        ),
+        privateKey: "",
+      });
+      await fetch("https://localhost:5552/fixture.json", {
+        client,
+      });
+    }, Deno.errors.InvalidData);
+  },
+);
+
+unitTest(
+  { perms: { read: true, net: true } },
+  async function fetchCustomClientPrivateKey(): Promise<
+    void
+  > {
+    const data = "Hello World";
+    const client = Deno.createHttpClient({
+      certChain: await Deno.readTextFile(
+        "cli/tests/testdata/tls/localhost.crt",
+      ),
+      privateKey: await Deno.readTextFile(
+        "cli/tests/testdata/tls/localhost.key",
+      ),
+      caData: await Deno.readTextFile("cli/tests/testdata/tls/RootCA.crt"),
+    });
+    const response = await fetch("https://localhost:5552/echo_server", {
+      client,
+      method: "POST",
+      body: new TextEncoder().encode(data),
+    });
+    assertEquals(
+      response.headers.get("user-agent"),
+      `Deno/${Deno.version.deno}`,
+    );
+    await response.text();
+    client.close();
+  },
+);

ext/fetch/lib.rs

@@ -62,6 +62,7 @@ pub fn init<P: FetchPermissions + 'static>(
   proxy: Option<Proxy>,
   request_builder_hook: Option<fn(RequestBuilder) -> RequestBuilder>,
   unsafely_ignore_certificate_errors: Option<Vec<String>>,
+  client_cert_chain_and_key: Option<(String, String)>,
 ) -> Extension {
   Extension::builder()
     .js(include_js_files!(
@@ -90,6 +91,7 @@ pub fn init<P: FetchPermissions + 'static>(
           None,
           proxy.clone(),
           unsafely_ignore_certificate_errors.clone(),
+          client_cert_chain_and_key.clone(),
         )
         .unwrap()
       });
@@ -100,6 +102,7 @@ pub fn init<P: FetchPermissions + 'static>(
         request_builder_hook,
         unsafely_ignore_certificate_errors: unsafely_ignore_certificate_errors
           .clone(),
+        client_cert_chain_and_key: client_cert_chain_and_key.clone(),
       });
       Ok(())
     })
@@ -112,6 +115,7 @@ pub struct HttpClientDefaults {
   pub proxy: Option<Proxy>,
   pub request_builder_hook: Option<fn(RequestBuilder) -> RequestBuilder>,
   pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
+  pub client_cert_chain_and_key: Option<(String, String)>,
 }
 
 pub trait FetchPermissions {
@@ -508,6 +512,8 @@ pub struct CreateHttpClientOptions {
   ca_file: Option<String>,
   ca_data: Option<ByteString>,
   proxy: Option<Proxy>,
+  cert_chain: Option<String>,
+  private_key: Option<String>,
 }
 
 pub fn op_create_http_client<FP>(
@@ -529,6 +535,21 @@ where
     permissions.check_net_url(&url)?;
   }
 
+  let client_cert_chain_and_key = {
+    if args.cert_chain.is_some() || args.private_key.is_some() {
+      let cert_chain = args
+        .cert_chain
+        .ok_or_else(|| type_error("No certificate chain provided"))?;
+      let private_key = args
+        .private_key
+        .ok_or_else(|| type_error("No private key provided"))?;
+
+      Some((cert_chain, private_key))
+    } else {
+      None
+    }
+  };
+
   let defaults = state.borrow::<HttpClientDefaults>();
   let cert_data =
     get_cert_data(args.ca_file.as_deref(), args.ca_data.as_deref())?;
@@ -539,8 +560,8 @@ where
     cert_data,
     args.proxy,
     defaults.unsafely_ignore_certificate_errors.clone(),
-  )
+    client_cert_chain_and_key,
-  .unwrap();
+  )?;
 
   let rid = state.resource_table.add(HttpClientResource::new(client));
   Ok(rid)

ext/net/ops_tls.rs

@@ -37,9 +37,8 @@ use deno_core::RcRef;
 use deno_core::Resource;
 use deno_core::ResourceId;
 use deno_tls::create_client_config;
-use deno_tls::rustls::internal::pemfile::certs;
+use deno_tls::load_certs;
-use deno_tls::rustls::internal::pemfile::pkcs8_private_keys;
+use deno_tls::load_private_keys;
-use deno_tls::rustls::internal::pemfile::rsa_private_keys;
 use deno_tls::rustls::Certificate;
 use deno_tls::rustls::ClientConfig;
 use deno_tls::rustls::ClientSession;
@@ -58,7 +57,6 @@ use std::cell::RefCell;
 use std::convert::From;
 use std::fs::File;
 use std::io;
-use std::io::BufRead;
 use std::io::BufReader;
 use std::io::ErrorKind;
 use std::ops::Deref;
@@ -862,58 +860,12 @@ where
   })
 }
 
-fn load_certs(reader: &mut dyn BufRead) -> Result<Vec<Certificate>, AnyError> {
-  let certs = certs(reader)
-    .map_err(|_| custom_error("InvalidData", "Unable to decode certificate"))?;
-
-  if certs.is_empty() {
-    let e = custom_error("InvalidData", "No certificates found in cert file");
-    return Err(e);
-  }
-
-  Ok(certs)
-}
-
 fn load_certs_from_file(path: &str) -> Result<Vec<Certificate>, AnyError> {
   let cert_file = File::open(path)?;
   let reader = &mut BufReader::new(cert_file);
   load_certs(reader)
 }
 
-fn key_decode_err() -> AnyError {
-  custom_error("InvalidData", "Unable to decode key")
-}
-
-fn key_not_found_err() -> AnyError {
-  custom_error("InvalidData", "No keys found in key file")
-}
-
-/// Starts with -----BEGIN RSA PRIVATE KEY-----
-fn load_rsa_keys(mut bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
-  let keys = rsa_private_keys(&mut bytes).map_err(|_| key_decode_err())?;
-  Ok(keys)
-}
-
-/// Starts with -----BEGIN PRIVATE KEY-----
-fn load_pkcs8_keys(mut bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
-  let keys = pkcs8_private_keys(&mut bytes).map_err(|_| key_decode_err())?;
-  Ok(keys)
-}
-
-fn load_private_keys(bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
-  let mut keys = load_rsa_keys(bytes)?;
-
-  if keys.is_empty() {
-    keys = load_pkcs8_keys(bytes)?;
-  }
-
-  if keys.is_empty() {
-    return Err(key_not_found_err());
-  }
-
-  Ok(keys)
-}
-
 fn load_private_keys_from_file(
   path: &str,
 ) -> Result<Vec<PrivateKey>, AnyError> {

ext/tls/lib.rs

@@ -7,6 +7,7 @@ pub use webpki;
 pub use webpki_roots;
 
 use deno_core::error::anyhow;
+use deno_core::error::custom_error;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::parking_lot::Mutex;
@@ -17,9 +18,13 @@ use reqwest::header::USER_AGENT;
 use reqwest::redirect::Policy;
 use reqwest::Client;
 use rustls::internal::msgs::handshake::DigitallySignedStruct;
+use rustls::internal::pemfile::certs;
+use rustls::internal::pemfile::pkcs8_private_keys;
+use rustls::internal::pemfile::rsa_private_keys;
 use rustls::Certificate;
 use rustls::ClientConfig;
 use rustls::HandshakeSignatureValid;
+use rustls::PrivateKey;
 use rustls::RootCertStore;
 use rustls::ServerCertVerified;
 use rustls::ServerCertVerifier;
@@ -28,6 +33,7 @@ use rustls::TLSError;
 use rustls::WebPKIVerifier;
 use serde::Deserialize;
 use std::collections::HashMap;
+use std::io::BufRead;
 use std::io::BufReader;
 use std::io::Cursor;
 use std::sync::Arc;
@@ -156,6 +162,54 @@ pub fn create_client_config(
   Ok(tls_config)
 }
 
+pub fn load_certs(
+  reader: &mut dyn BufRead,
+) -> Result<Vec<Certificate>, AnyError> {
+  let certs = certs(reader)
+    .map_err(|_| custom_error("InvalidData", "Unable to decode certificate"))?;
+
+  if certs.is_empty() {
+    let e = custom_error("InvalidData", "No certificates found in cert file");
+    return Err(e);
+  }
+
+  Ok(certs)
+}
+
+fn key_decode_err() -> AnyError {
+  custom_error("InvalidData", "Unable to decode key")
+}
+
+fn key_not_found_err() -> AnyError {
+  custom_error("InvalidData", "No keys found in key file")
+}
+
+/// Starts with -----BEGIN RSA PRIVATE KEY-----
+fn load_rsa_keys(mut bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
+  let keys = rsa_private_keys(&mut bytes).map_err(|_| key_decode_err())?;
+  Ok(keys)
+}
+
+/// Starts with -----BEGIN PRIVATE KEY-----
+fn load_pkcs8_keys(mut bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
+  let keys = pkcs8_private_keys(&mut bytes).map_err(|_| key_decode_err())?;
+  Ok(keys)
+}
+
+pub fn load_private_keys(bytes: &[u8]) -> Result<Vec<PrivateKey>, AnyError> {
+  let mut keys = load_rsa_keys(bytes)?;
+
+  if keys.is_empty() {
+    keys = load_pkcs8_keys(bytes)?;
+  }
+
+  if keys.is_empty() {
+    return Err(key_not_found_err());
+  }
+
+  Ok(keys)
+}
+
 /// Create new instance of async reqwest::Client. This client supports
 /// proxies and doesn't follow redirects.
 pub fn create_http_client(
@@ -164,12 +218,26 @@ pub fn create_http_client(
   ca_data: Option<Vec<u8>>,
   proxy: Option<Proxy>,
   unsafely_ignore_certificate_errors: Option<Vec<String>>,
+  client_cert_chain_and_key: Option<(String, String)>,
 ) -> Result<Client, AnyError> {
-  let tls_config = create_client_config(
+  let mut tls_config = create_client_config(
     root_cert_store,
     ca_data,
     unsafely_ignore_certificate_errors,
   )?;
+
+  if let Some((cert_chain, private_key)) = client_cert_chain_and_key {
+    // The `remove` is safe because load_private_keys checks that there is at least one key.
+    let private_key = load_private_keys(private_key.as_bytes())?.remove(0);
+
+    tls_config
+      .set_single_client_cert(
+        load_certs(&mut cert_chain.as_bytes())?,
+        private_key,
+      )
+      .expect("invalid client key or certificate");
+  }
+
   let mut headers = HeaderMap::new();
   headers.insert(USER_AGENT, user_agent.parse().unwrap());
   let mut builder = Client::builder()

runtime/build.rs

@@ -54,6 +54,7 @@ mod not_docs {
         None,
         None,
         None,
+        None,
       ),
       deno_websocket::init::<deno_websocket::NoWebSocketPermissions>(
         "".to_owned(),

runtime/web_worker.rs

@@ -320,6 +320,7 @@ impl WebWorker {
         None,
         None,
         options.unsafely_ignore_certificate_errors.clone(),
+        None,
       ),
       deno_websocket::init::<Permissions>(
         options.user_agent.clone(),

runtime/worker.rs

@@ -108,6 +108,7 @@ impl MainWorker {
         None,
         None,
         options.unsafely_ignore_certificate_errors.clone(),
+        None,
       ),
       deno_websocket::init::<Permissions>(
         options.user_agent.clone(),

test_util/src/lib.rs

@@ -59,6 +59,7 @@ const REDIRECT_ABSOLUTE_PORT: u16 = 4550;
 const AUTH_REDIRECT_PORT: u16 = 4551;
 const TLS_CLIENT_AUTH_PORT: u16 = 4552;
 const HTTPS_PORT: u16 = 5545;
+const HTTPS_CLIENT_AUTH_PORT: u16 = 5552;
 const WS_PORT: u16 = 4242;
 const WSS_PORT: u16 = 4243;
 const WS_CLOSE_PORT: u16 = 4244;
@@ -898,6 +899,62 @@ async fn wrap_main_https_server() {
   }
 }
 
+async fn wrap_client_auth_https_server() {
+  let main_server_https_addr =
+    SocketAddr::from(([127, 0, 0, 1], HTTPS_CLIENT_AUTH_PORT));
+  let cert_file = "tls/localhost.crt";
+  let key_file = "tls/localhost.key";
+  let ca_cert_file = "tls/RootCA.pem";
+  let tls_config = get_tls_config(cert_file, key_file, ca_cert_file)
+    .await
+    .unwrap();
+  loop {
+    let tcp = TcpListener::bind(&main_server_https_addr)
+      .await
+      .expect("Cannot bind TCP");
+    println!("ready: https_client_auth on :{:?}", HTTPS_CLIENT_AUTH_PORT); // Eye catcher for HttpServerCount
+    let tls_acceptor = TlsAcceptor::from(tls_config.clone());
+    // Prepare a long-running future stream to accept and serve cients.
+    let incoming_tls_stream = async_stream::stream! {
+      loop {
+          let (socket, _) = tcp.accept().await?;
+
+          match tls_acceptor.accept(socket).await {
+            Ok(mut tls_stream) => {
+              let (_, tls_session) = tls_stream.get_mut();
+              // We only need to check for the presence of client certificates
+              // here. Rusttls ensures that they are valid and signed by the CA.
+              match tls_session.get_peer_certificates() {
+                Some(_certs) => { yield Ok(tls_stream); },
+                None => { eprintln!("https_client_auth: no valid client certificate"); },
+              };
+            }
+
+            Err(e) => {
+              eprintln!("https-client-auth accept error: {:?}", e);
+              yield Err(e);
+            }
+          }
+
+      }
+    }
+    .boxed();
+
+    let main_server_https_svc = make_service_fn(|_| async {
+      Ok::<_, Infallible>(service_fn(main_server))
+    });
+    let main_server_https = Server::builder(HyperAcceptor {
+      acceptor: incoming_tls_stream,
+    })
+    .serve(main_server_https_svc);
+
+    //continue to prevent TLS error stopping the server
+    if main_server_https.await.is_err() {
+      continue;
+    }
+  }
+}
+
 // Use the single-threaded scheduler. The hyper server is used as a point of
 // comparison for the (single-threaded!) benchmarks in cli/bench. We're not
 // comparing apples to apples if we use the default multi-threaded scheduler.
@@ -922,7 +979,7 @@ pub async fn run_all_servers() {
   let ws_close_server_fut = run_ws_close_server(&ws_close_addr);
 
   let tls_client_auth_server_fut = run_tls_client_auth_server();
-
+  let client_auth_server_https_fut = wrap_client_auth_https_server();
   let main_server_fut = wrap_main_server();
   let main_server_https_fut = wrap_main_https_server();
 
@@ -940,6 +997,7 @@ pub async fn run_all_servers() {
       abs_redirect_server_fut,
       main_server_fut,
       main_server_https_fut,
+      client_auth_server_https_fut,
     )
   }
   .boxed();