development/deno
development/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno synced 2024-08-27 12:00:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(npm): use an http client with connection pool (#16705)

Browse source 
Should make downloading npm packages faster and more reliable.
This commit is contained in:
David Sherret 2022-11-18 17:28:14 -05:00 committed by GitHub
parent 6962808f4b
commit 763d492ed6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 454 additions and 422 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

64
cli/file_fetcher.rs
View file

@ -3,13 +3,12 @@
use crate::auth_tokens::AuthTokens; use crate::auth_tokens::AuthTokens;
use crate::colors; use crate::colors;
use crate::http_cache::HttpCache; use crate::http_cache::HttpCache;
use crate::http_util::fetch_once;
use crate::http_util::CacheSemantics; use crate::http_util::CacheSemantics;
use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceArgs;
use crate::http_util::FetchOnceResult; use crate::http_util::FetchOnceResult;
use crate::http_util::HttpClient;
use crate::progress_bar::ProgressBar; use crate::progress_bar::ProgressBar;
use crate::text_encoding; use crate::text_encoding;
use crate::version::get_user_agent;
use data_url::DataUrl; use data_url::DataUrl;
use deno_ast::MediaType; use deno_ast::MediaType;
@ -22,8 +21,6 @@ use deno_core::futures;
use deno_core::futures::future::FutureExt; use deno_core::futures::future::FutureExt;
use deno_core::parking_lot::Mutex; use deno_core::parking_lot::Mutex;
use deno_core::ModuleSpecifier; use deno_core::ModuleSpecifier;
use deno_runtime::deno_fetch::create_http_client;
use deno_runtime::deno_fetch::reqwest;
use deno_runtime::deno_tls::rustls; use deno_runtime::deno_tls::rustls;
use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::rustls::RootCertStore;
use deno_runtime::deno_tls::rustls_native_certs::load_native_certs; use deno_runtime::deno_tls::rustls_native_certs::load_native_certs;
@ -333,7 +330,7 @@ pub struct FileFetcher {
cache: FileCache, cache: FileCache,
cache_setting: CacheSetting, cache_setting: CacheSetting,
pub http_cache: HttpCache, pub http_cache: HttpCache,
http_client: reqwest::Client, http_client: HttpClient,
blob_store: BlobStore, blob_store: BlobStore,
download_log_level: log::Level, download_log_level: log::Level,
progress_bar: Option<ProgressBar>, progress_bar: Option<ProgressBar>,
@ -344,9 +341,8 @@ impl FileFetcher {
http_cache: HttpCache, http_cache: HttpCache,
cache_setting: CacheSetting, cache_setting: CacheSetting,
allow_remote: bool, allow_remote: bool,
root_cert_store: Option<RootCertStore>, http_client: HttpClient,
blob_store: BlobStore, blob_store: BlobStore,
unsafely_ignore_certificate_errors: Option<Vec<String>>,
progress_bar: Option<ProgressBar>, progress_bar: Option<ProgressBar>,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
Ok(Self { Ok(Self {
@ -355,14 +351,7 @@ impl FileFetcher {
cache: Default::default(), cache: Default::default(),
cache_setting, cache_setting,
http_cache, http_cache,
http_client: create_http_client( http_client,
get_user_agent(),
root_cert_store,
vec![],
None,
unsafely_ignore_certificate_errors,
None,
)?,
blob_store, blob_store,
download_log_level: log::Level::Info, download_log_level: log::Level::Info,
progress_bar, progress_bar,
@ -628,14 +617,14 @@ impl FileFetcher {
let file_fetcher = self.clone(); let file_fetcher = self.clone();
// A single pass of fetch either yields code or yields a redirect. // A single pass of fetch either yields code or yields a redirect.
async move { async move {
match fetch_once(FetchOnceArgs { match client
client, .fetch_once(FetchOnceArgs {
url: specifier.clone(), url: specifier.clone(),
maybe_accept: maybe_accept.clone(), maybe_accept: maybe_accept.clone(),
maybe_etag, maybe_etag,
maybe_auth_token, maybe_auth_token,
}) })
.await? .await?
{ {
FetchOnceResult::NotModified => { FetchOnceResult::NotModified => {
let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap(); let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap();
@ -765,6 +754,8 @@ impl FileFetcher {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use crate::http_util::HttpClient;
use super::*; use super::*;
use deno_core::error::get_custom_error_class; use deno_core::error::get_custom_error_class;
use deno_core::resolve_url; use deno_core::resolve_url;
@ -793,10 +784,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
cache_setting, cache_setting,
true, true,
None, HttpClient::new(None, None).unwrap(),
blob_store.clone(), blob_store.clone(),
None, None,
None,
) )
.unwrap(); .unwrap();
(file_fetcher, temp_dir, blob_store) (file_fetcher, temp_dir, blob_store)
@ -1232,10 +1222,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::ReloadAll, CacheSetting::ReloadAll,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let result = file_fetcher let result = file_fetcher
@ -1259,10 +1248,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let specifier = let specifier =
@ -1287,10 +1275,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let result = file_fetcher_02 let result = file_fetcher_02
@ -1431,10 +1418,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let specifier = let specifier =
@ -1461,10 +1447,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let result = file_fetcher_02 let result = file_fetcher_02
@ -1562,10 +1547,9 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
false, false,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let specifier = let specifier =
@ -1589,20 +1573,18 @@ mod tests {
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Only, CacheSetting::Only,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let file_fetcher_02 = FileFetcher::new( let file_fetcher_02 = FileFetcher::new(
HttpCache::new(&location), HttpCache::new(&location),
CacheSetting::Use, CacheSetting::Use,
true, true,
None, HttpClient::new(None, None).unwrap(),
BlobStore::default(), BlobStore::default(),
None, None,
None,
) )
.unwrap(); .unwrap();
let specifier = let specifier =

10
cli/http_cache.rs
View file

@ -73,11 +73,6 @@ pub fn url_to_filename(url: &Url) -> Option<PathBuf> {
Some(cache_filename) Some(cache_filename)
} }
#[derive(Debug, Clone, Default)]
pub struct HttpCache {
pub location: PathBuf,
}
#[derive(Serialize, Deserialize)] #[derive(Serialize, Deserialize)]
pub struct Metadata { pub struct Metadata {
pub headers: HeadersMap, pub headers: HeadersMap,
@ -107,6 +102,11 @@ impl Metadata {
} }
} }
#[derive(Debug, Clone, Default)]
pub struct HttpCache {
pub location: PathBuf,
}
impl HttpCache { impl HttpCache {
/// Returns a new instance. /// Returns a new instance.
/// ///

591
cli/http_util.rs
View file

@ -1,5 +1,6 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use crate::auth_tokens::AuthToken; use crate::auth_tokens::AuthToken;
use crate::version::get_user_agent;
use cache_control::Cachability; use cache_control::Cachability;
use cache_control::CacheControl; use cache_control::CacheControl;
@ -8,13 +9,15 @@ use deno_core::error::custom_error;
use deno_core::error::generic_error; use deno_core::error::generic_error;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_core::url::Url; use deno_core::url::Url;
use deno_runtime::deno_fetch::create_http_client;
use deno_runtime::deno_fetch::reqwest;
use deno_runtime::deno_fetch::reqwest::header::HeaderValue; use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
use deno_runtime::deno_fetch::reqwest::header::ACCEPT; use deno_runtime::deno_fetch::reqwest::header::ACCEPT;
use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH; use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
use deno_runtime::deno_fetch::reqwest::header::LOCATION; use deno_runtime::deno_fetch::reqwest::header::LOCATION;
use deno_runtime::deno_fetch::reqwest::Client;
use deno_runtime::deno_fetch::reqwest::StatusCode; use deno_runtime::deno_fetch::reqwest::StatusCode;
use deno_runtime::deno_tls::rustls::RootCertStore;
use log::debug; use log::debug;
use std::collections::HashMap; use std::collections::HashMap;
use std::time::Duration; use std::time::Duration;
@ -208,97 +211,125 @@ pub enum FetchOnceResult {
#[derive(Debug)] #[derive(Debug)]
pub struct FetchOnceArgs { pub struct FetchOnceArgs {
pub client: Client,
pub url: Url, pub url: Url,
pub maybe_accept: Option<String>, pub maybe_accept: Option<String>,
pub maybe_etag: Option<String>, pub maybe_etag: Option<String>,
pub maybe_auth_token: Option<AuthToken>, pub maybe_auth_token: Option<AuthToken>,
} }
/// Asynchronously fetches the given HTTP URL one pass only. #[derive(Debug, Clone)]
/// If no redirect is present and no error occurs, pub struct HttpClient(reqwest::Client);
/// yields Code(ResultPayload).
/// If redirect occurs, does not follow and
/// yields Redirect(url).
pub async fn fetch_once(
args: FetchOnceArgs,
) -> Result<FetchOnceResult, AnyError> {
let mut request = args.client.get(args.url.clone());
if let Some(etag) = args.maybe_etag { impl HttpClient {
let if_none_match_val = HeaderValue::from_str(&etag)?; pub fn new(
request = request.header(IF_NONE_MATCH, if_none_match_val); root_cert_store: Option<RootCertStore>,
} unsafely_ignore_certificate_errors: Option<Vec<String>>,
if let Some(auth_token) = args.maybe_auth_token { ) -> Result<Self, AnyError> {
let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; Ok(HttpClient::from_client(create_http_client(
request = request.header(AUTHORIZATION, authorization_val); get_user_agent(),
} root_cert_store,
if let Some(accept) = args.maybe_accept { vec![],
let accepts_val = HeaderValue::from_str(&accept)?; None,
request = request.header(ACCEPT, accepts_val); unsafely_ignore_certificate_errors,
} None,
let response = request.send().await?; )?))
if response.status() == StatusCode::NOT_MODIFIED {
return Ok(FetchOnceResult::NotModified);
} }
let mut headers_: HashMap<String, String> = HashMap::new(); pub fn from_client(client: reqwest::Client) -> Self {
let headers = response.headers(); Self(client)
if let Some(warning) = headers.get("X-Deno-Warning") {
eprintln!(
"{} {}",
crate::colors::yellow("Warning"),
warning.to_str().unwrap()
);
} }
for key in headers.keys() { pub fn get<U: reqwest::IntoUrl>(&self, url: U) -> reqwest::RequestBuilder {
let key_str = key.to_string(); self.0.get(url)
let values = headers.get_all(key);
let values_str = values
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect::<Vec<String>>()
.join(",");
headers_.insert(key_str, values_str);
} }
if response.status().is_redirection() { /// Asynchronously fetches the given HTTP URL one pass only.
if let Some(location) = response.headers().get(LOCATION) { /// If no redirect is present and no error occurs,
let location_string = location.to_str().unwrap(); /// yields Code(ResultPayload).
debug!("Redirecting to {:?}...", &location_string); /// If redirect occurs, does not follow and
let new_url = resolve_url_from_location(&args.url, location_string); /// yields Redirect(url).
return Ok(FetchOnceResult::Redirect(new_url, headers_)); pub async fn fetch_once(
} else { &self,
return Err(generic_error(format!( args: FetchOnceArgs,
"Redirection from '{}' did not provide location header", ) -> Result<FetchOnceResult, AnyError> {
args.url let mut request = self.get(args.url.clone());
)));
if let Some(etag) = args.maybe_etag {
let if_none_match_val = HeaderValue::from_str(&etag)?;
request = request.header(IF_NONE_MATCH, if_none_match_val);
} }
if let Some(auth_token) = args.maybe_auth_token {
let authorization_val = HeaderValue::from_str(&auth_token.to_string())?;
request = request.header(AUTHORIZATION, authorization_val);
}
if let Some(accept) = args.maybe_accept {
let accepts_val = HeaderValue::from_str(&accept)?;
request = request.header(ACCEPT, accepts_val);
}
let response = request.send().await?;
if response.status() == StatusCode::NOT_MODIFIED {
return Ok(FetchOnceResult::NotModified);
}
let mut result_headers = HashMap::new();
let response_headers = response.headers();
if let Some(warning) = response_headers.get("X-Deno-Warning") {
eprintln!(
"{} {}",
crate::colors::yellow("Warning"),
warning.to_str().unwrap()
);
}
for key in response_headers.keys() {
let key_str = key.to_string();
let values = response_headers.get_all(key);
let values_str = values
.iter()
.map(|e| e.to_str().unwrap().to_string())
.collect::<Vec<String>>()
.join(",");
result_headers.insert(key_str, values_str);
}
if response.status().is_redirection() {
if let Some(location) = response.headers().get(LOCATION) {
let location_string = location.to_str().unwrap();
debug!("Redirecting to {:?}...", &location_string);
let new_url = resolve_url_from_location(&args.url, location_string);
return Ok(FetchOnceResult::Redirect(new_url, result_headers));
} else {
return Err(generic_error(format!(
"Redirection from '{}' did not provide location header",
args.url
)));
}
}
if response.status().is_client_error()
|| response.status().is_server_error()
{
let err = if response.status() == StatusCode::NOT_FOUND {
custom_error(
"NotFound",
format!("Import '{}' failed, not found.", args.url),
)
} else {
generic_error(format!(
"Import '{}' failed: {}",
args.url,
response.status()
))
};
return Err(err);
}
let body = response.bytes().await?.to_vec();
Ok(FetchOnceResult::Code(body, result_headers))
} }
if response.status().is_client_error() || response.status().is_server_error()
{
let err = if response.status() == StatusCode::NOT_FOUND {
custom_error(
"NotFound",
format!("Import '{}' failed, not found.", args.url),
)
} else {
generic_error(format!(
"Import '{}' failed: {}",
args.url,
response.status()
))
};
return Err(err);
}
let body = response.bytes().await?.to_vec();
Ok(FetchOnceResult::Code(body, headers_))
} }
#[cfg(test)] #[cfg(test)]
@ -308,16 +339,18 @@ mod tests {
use deno_runtime::deno_fetch::create_http_client; use deno_runtime::deno_fetch::create_http_client;
use std::fs::read; use std::fs::read;
fn create_test_client() -> Client { fn create_test_client() -> HttpClient {
create_http_client( HttpClient::from_client(
"test_client".to_string(), create_http_client(
None, "test_client".to_string(),
vec![], None,
None, vec![],
None, None,
None, None,
None,
)
.unwrap(),
) )
.unwrap()
} }
#[tokio::test] #[tokio::test]
@ -326,14 +359,14 @@ mod tests {
// Relies on external http server. See target/debug/test_server // Relies on external http server. See target/debug/test_server
let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap(); let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(headers.get("content-type").unwrap(), "application/json"); assert_eq!(headers.get("content-type").unwrap(), "application/json");
@ -351,14 +384,14 @@ mod tests {
let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped") let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped")
.unwrap(); .unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
assert_eq!( assert_eq!(
@ -377,14 +410,14 @@ mod tests {
let _http_server_guard = test_util::http_server(); let _http_server_guard = test_util::http_server();
let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client: client.clone(), .fetch_once(FetchOnceArgs {
url: url.clone(), url: url.clone(),
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@ -397,14 +430,14 @@ mod tests {
panic!(); panic!();
} }
let res = fetch_once(FetchOnceArgs { let res = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_etag: Some("33a64df551425fcc55e".to_string()),
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
assert_eq!(res.unwrap(), FetchOnceResult::NotModified); assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
} }
@ -415,14 +448,14 @@ mod tests {
let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli") let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli")
.unwrap(); .unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@ -443,14 +476,14 @@ mod tests {
// Relies on external http server. See target/debug/test_server // Relies on external http server. See target/debug/test_server
let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap(); let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: Some("application/json".to_string()), maybe_accept: Some("application/json".to_string()),
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(body, _)) = result { if let Ok(FetchOnceResult::Code(body, _)) = result {
assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes()); assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes());
} else { } else {
@ -467,14 +500,14 @@ mod tests {
let target_url = let target_url =
Url::parse("http://localhost:4545/assets/fixture.json").unwrap(); Url::parse("http://localhost:4545/assets/fixture.json").unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Redirect(url, _)) = result { if let Ok(FetchOnceResult::Redirect(url, _)) = result {
assert_eq!(url, target_url); assert_eq!(url, target_url);
} else { } else {
@ -526,29 +559,31 @@ mod tests {
// Relies on external http server. See target/debug/test_server // Relies on external http server. See target/debug/test_server
let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap(); let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap();
let client = create_http_client( let client = HttpClient::from_client(
version::get_user_agent(), create_http_client(
None, version::get_user_agent(),
vec![read( None,
test_util::testdata_path() vec![read(
.join("tls/RootCA.pem") test_util::testdata_path()
.to_str() .join("tls/RootCA.pem")
.unwrap(), .to_str()
.unwrap(),
)
.unwrap()],
None,
None,
None,
) )
.unwrap()], .unwrap(),
None, );
None, let result = client
None, .fetch_once(FetchOnceArgs {
) url,
.unwrap(); maybe_accept: None,
let result = fetch_once(FetchOnceArgs { maybe_etag: None,
client, maybe_auth_token: None,
url, })
maybe_accept: None, .await;
maybe_etag: None,
maybe_auth_token: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(headers.get("content-type").unwrap(), "application/json"); assert_eq!(headers.get("content-type").unwrap(), "application/json");
@ -564,24 +599,26 @@ mod tests {
let _http_server_guard = test_util::http_server(); let _http_server_guard = test_util::http_server();
// Relies on external http server with a valid mozilla root CA cert. // Relies on external http server with a valid mozilla root CA cert.
let url = Url::parse("https://deno.land").unwrap(); let url = Url::parse("https://deno.land").unwrap();
let client = create_http_client( let client = HttpClient::from_client(
version::get_user_agent(), create_http_client(
None, // This will load mozilla certs by default version::get_user_agent(),
vec![], None, // This will load mozilla certs by default
None, vec![],
None, None,
None, None,
) None,
.unwrap(); )
.unwrap(),
);
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
println!("{:?}", result); println!("{:?}", result);
if let Ok(FetchOnceResult::Code(body, _headers)) = result { if let Ok(FetchOnceResult::Code(body, _headers)) = result {
@ -601,24 +638,20 @@ mod tests {
let _http_server_guard = test_util::http_server(); let _http_server_guard = test_util::http_server();
// Relies on external http server with a valid mozilla root CA cert. // Relies on external http server with a valid mozilla root CA cert.
let url = Url::parse("https://deno.land").unwrap(); let url = Url::parse("https://deno.land").unwrap();
let client = create_http_client( let client = HttpClient::new(
version::get_user_agent(),
Some(RootCertStore::empty()), // no certs loaded at all Some(RootCertStore::empty()), // no certs loaded at all
vec![],
None,
None,
None, None,
) )
.unwrap(); .unwrap();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
if let Ok(FetchOnceResult::Code(_body, _headers)) = result { if let Ok(FetchOnceResult::Code(_body, _headers)) = result {
// This test is expected to fail since to CA certs have been loaded // This test is expected to fail since to CA certs have been loaded
@ -633,29 +666,31 @@ mod tests {
let url = let url =
Url::parse("https://localhost:5545/run/import_compression/gziped") Url::parse("https://localhost:5545/run/import_compression/gziped")
.unwrap(); .unwrap();
let client = create_http_client( let client = HttpClient::from_client(
version::get_user_agent(), create_http_client(
None, version::get_user_agent(),
vec![read( None,
test_util::testdata_path() vec![read(
.join("tls/RootCA.pem") test_util::testdata_path()
.to_str() .join("tls/RootCA.pem")
.unwrap(), .to_str()
.unwrap(),
)
.unwrap()],
None,
None,
None,
) )
.unwrap()], .unwrap(),
None, );
None, let result = client
None, .fetch_once(FetchOnceArgs {
) url,
.unwrap(); maybe_accept: None,
let result = fetch_once(FetchOnceArgs { maybe_etag: None,
client, maybe_auth_token: None,
url, })
maybe_accept: None, .await;
maybe_etag: None,
maybe_auth_token: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
assert_eq!( assert_eq!(
@ -673,29 +708,31 @@ mod tests {
async fn test_fetch_with_cafile_with_etag() { async fn test_fetch_with_cafile_with_etag() {
let _http_server_guard = test_util::http_server(); let _http_server_guard = test_util::http_server();
let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap(); let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();
let client = create_http_client( let client = HttpClient::from_client(
version::get_user_agent(), create_http_client(
None, version::get_user_agent(),
vec![read( None,
test_util::testdata_path() vec![read(
.join("tls/RootCA.pem") test_util::testdata_path()
.to_str() .join("tls/RootCA.pem")
.unwrap(), .to_str()
.unwrap(),
)
.unwrap()],
None,
None,
None,
) )
.unwrap()], .unwrap(),
None, );
None, let result = client
None, .fetch_once(FetchOnceArgs {
) url: url.clone(),
.unwrap(); maybe_accept: None,
let result = fetch_once(FetchOnceArgs { maybe_etag: None,
client: client.clone(), maybe_auth_token: None,
url: url.clone(), })
maybe_accept: None, .await;
maybe_etag: None,
maybe_auth_token: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@ -709,14 +746,14 @@ mod tests {
panic!(); panic!();
} }
let res = fetch_once(FetchOnceArgs { let res = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: Some("33a64df551425fcc55e".to_string()), maybe_etag: Some("33a64df551425fcc55e".to_string()),
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
assert_eq!(res.unwrap(), FetchOnceResult::NotModified); assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
} }
@ -727,29 +764,31 @@ mod tests {
let url = let url =
Url::parse("https://localhost:5545/run/import_compression/brotli") Url::parse("https://localhost:5545/run/import_compression/brotli")
.unwrap(); .unwrap();
let client = create_http_client( let client = HttpClient::from_client(
version::get_user_agent(), create_http_client(
None, version::get_user_agent(),
vec![read( None,
test_util::testdata_path() vec![read(
.join("tls/RootCA.pem") test_util::testdata_path()
.to_str() .join("tls/RootCA.pem")
.unwrap(), .to_str()
.unwrap(),
)
.unwrap()],
None,
None,
None,
) )
.unwrap()], .unwrap(),
None, );
None, let result = client
None, .fetch_once(FetchOnceArgs {
) url,
.unwrap(); maybe_accept: None,
let result = fetch_once(FetchOnceArgs { maybe_etag: None,
client, maybe_auth_token: None,
url, })
maybe_accept: None, .await;
maybe_etag: None,
maybe_auth_token: None,
})
.await;
if let Ok(FetchOnceResult::Code(body, headers)) = result { if let Ok(FetchOnceResult::Code(body, headers)) = result {
assert!(!body.is_empty()); assert!(!body.is_empty());
assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');"); assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@ -770,14 +809,14 @@ mod tests {
let url_str = "http://127.0.0.1:4545/bad_redirect"; let url_str = "http://127.0.0.1:4545/bad_redirect";
let url = Url::parse(url_str).unwrap(); let url = Url::parse(url_str).unwrap();
let client = create_test_client(); let client = create_test_client();
let result = fetch_once(FetchOnceArgs { let result = client
client, .fetch_once(FetchOnceArgs {
url, url,
maybe_accept: None, maybe_accept: None,
maybe_etag: None, maybe_etag: None,
maybe_auth_token: None, maybe_auth_token: None,
}) })
.await; .await;
assert!(result.is_err()); assert!(result.is_err());
let err = result.unwrap_err(); let err = result.unwrap_err();
// Check that the error message contains the original URL // Check that the error message contains the original URL

141
cli/lsp/language_server.rs
View file

@ -50,7 +50,6 @@ use super::parent_process_checker;
use super::performance::Performance; use super::performance::Performance;
use super::refactor; use super::refactor;
use super::registries::ModuleRegistry; use super::registries::ModuleRegistry;
use super::registries::ModuleRegistryOptions;
use super::testing; use super::testing;
use super::text; use super::text;
use super::tsc; use super::tsc;
@ -65,10 +64,13 @@ use crate::args::FmtConfig;
use crate::args::LintConfig; use crate::args::LintConfig;
use crate::args::TsConfig; use crate::args::TsConfig;
use crate::deno_dir; use crate::deno_dir;
use crate::deno_dir::DenoDir;
use crate::file_fetcher::get_root_cert_store;
use crate::file_fetcher::get_source_from_data_url; use crate::file_fetcher::get_source_from_data_url;
use crate::file_fetcher::CacheSetting; use crate::file_fetcher::CacheSetting;
use crate::fs_util; use crate::fs_util;
use crate::graph_util::graph_valid; use crate::graph_util::graph_valid;
use crate::http_util::HttpClient;
use crate::npm::NpmCache; use crate::npm::NpmCache;
use crate::npm::NpmPackageResolver; use crate::npm::NpmPackageResolver;
use crate::npm::RealNpmRegistryApi; use crate::npm::RealNpmRegistryApi;
@ -235,17 +237,43 @@ impl LanguageServer {
} }
} }
fn create_lsp_npm_resolver(
dir: &DenoDir,
http_client: HttpClient,
) -> NpmPackageResolver {
let registry_url = RealNpmRegistryApi::default_url();
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
// the cache from being filled with lots of packages while
// the user is typing.
let cache_setting = CacheSetting::Only;
let progress_bar = ProgressBar::default();
let npm_cache = NpmCache::from_deno_dir(
dir,
cache_setting.clone(),
http_client.clone(),
progress_bar.clone(),
);
let api = RealNpmRegistryApi::new(
registry_url,
npm_cache.clone(),
cache_setting,
http_client,
progress_bar,
);
NpmPackageResolver::new(npm_cache, api, false, None)
}
impl Inner { impl Inner {
fn new(client: Client) -> Self { fn new(client: Client) -> Self {
let maybe_custom_root = env::var("DENO_DIR").map(String::into).ok(); let maybe_custom_root = env::var("DENO_DIR").map(String::into).ok();
let dir = deno_dir::DenoDir::new(maybe_custom_root) let dir = deno_dir::DenoDir::new(maybe_custom_root)
.expect("could not access DENO_DIR"); .expect("could not access DENO_DIR");
let module_registries_location = dir.root.join(REGISTRIES_PATH); let module_registries_location = dir.root.join(REGISTRIES_PATH);
let module_registries = ModuleRegistry::new( let http_client = HttpClient::new(None, None).unwrap();
&module_registries_location, let module_registries =
ModuleRegistryOptions::default(), ModuleRegistry::new(&module_registries_location, http_client.clone())
) .unwrap();
.expect("could not create module registries");
let location = dir.root.join(CACHE_PATH); let location = dir.root.join(CACHE_PATH);
let documents = Documents::new(&location); let documents = Documents::new(&location);
let cache_metadata = cache::CacheMetadata::new(&location); let cache_metadata = cache::CacheMetadata::new(&location);
@ -258,25 +286,7 @@ impl Inner {
ts_server.clone(), ts_server.clone(),
); );
let assets = Assets::new(ts_server.clone()); let assets = Assets::new(ts_server.clone());
let registry_url = RealNpmRegistryApi::default_url(); let npm_resolver = create_lsp_npm_resolver(&dir, http_client);
// Use an "only" cache setting in order to make the
// user do an explicit "cache" command and prevent
// the cache from being filled with lots of packages while
// the user is typing.
let cache_setting = CacheSetting::Only;
let progress_bar = ProgressBar::default();
let npm_cache = NpmCache::from_deno_dir(
&dir,
cache_setting.clone(),
progress_bar.clone(),
);
let api = RealNpmRegistryApi::new(
registry_url,
npm_cache.clone(),
cache_setting,
progress_bar,
);
let npm_resolver = NpmPackageResolver::new(npm_cache, api, false, None);
Self { Self {
assets, assets,
@ -498,36 +508,48 @@ impl Inner {
None None
}; };
if self.maybe_cache_path != maybe_cache_path { if self.maybe_cache_path != maybe_cache_path {
let maybe_custom_root = maybe_cache_path self.recreate_http_client_and_dependents(maybe_cache_path)?;
.clone()
.or_else(|| env::var("DENO_DIR").map(String::into).ok());
let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
let module_registries_location = dir.root.join(REGISTRIES_PATH);
let workspace_settings = self.config.get_workspace_settings();
let maybe_root_path = self
.config
.root_uri
.as_ref()
.and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
self.module_registries = ModuleRegistry::new(
&module_registries_location,
ModuleRegistryOptions {
maybe_root_path,
maybe_ca_stores: workspace_settings.certificate_stores.clone(),
maybe_ca_file: workspace_settings.tls_certificate.clone(),
unsafely_ignore_certificate_errors: workspace_settings
.unsafely_ignore_certificate_errors,
},
)?;
self.module_registries_location = module_registries_location;
let location = dir.root.join(CACHE_PATH);
self.documents.set_location(&location);
self.cache_metadata.set_location(&location);
self.maybe_cache_path = maybe_cache_path;
} }
Ok(()) Ok(())
} }
/// Recreates the http client and all dependent structs.
fn recreate_http_client_and_dependents(
&mut self,
new_cache_path: Option<PathBuf>,
) -> Result<(), AnyError> {
let maybe_custom_root = new_cache_path
.clone()
.or_else(|| env::var("DENO_DIR").map(String::into).ok());
let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
let workspace_settings = self.config.get_workspace_settings();
let maybe_root_path = self
.config
.root_uri
.as_ref()
.and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
let root_cert_store = Some(get_root_cert_store(
maybe_root_path,
workspace_settings.certificate_stores.clone(),
workspace_settings.tls_certificate.clone(),
)?);
let client = HttpClient::new(
root_cert_store,
workspace_settings.unsafely_ignore_certificate_errors,
)?;
let module_registries_location = dir.root.join(REGISTRIES_PATH);
self.module_registries =
ModuleRegistry::new(&module_registries_location, client.clone())?;
self.module_registries_location = module_registries_location;
self.npm_resolver = create_lsp_npm_resolver(&dir, client);
// update the cache path
let location = dir.root.join(CACHE_PATH);
self.documents.set_location(&location);
self.cache_metadata.set_location(&location);
self.maybe_cache_path = new_cache_path;
Ok(())
}
pub async fn update_import_map(&mut self) -> Result<(), AnyError> { pub async fn update_import_map(&mut self) -> Result<(), AnyError> {
let mark = self.performance.mark("update_import_map", None::<()>); let mark = self.performance.mark("update_import_map", None::<()>);
@ -627,23 +649,8 @@ impl Inner {
async fn update_registries(&mut self) -> Result<(), AnyError> { async fn update_registries(&mut self) -> Result<(), AnyError> {
let mark = self.performance.mark("update_registries", None::<()>); let mark = self.performance.mark("update_registries", None::<()>);
self.recreate_http_client_and_dependents(self.maybe_cache_path.clone())?;
let workspace_settings = self.config.get_workspace_settings(); let workspace_settings = self.config.get_workspace_settings();
let maybe_root_path = self
.config
.root_uri
.as_ref()
.and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
self.module_registries = ModuleRegistry::new(
&self.module_registries_location,
ModuleRegistryOptions {
maybe_root_path,
maybe_ca_stores: workspace_settings.certificate_stores.clone(),
maybe_ca_file: workspace_settings.tls_certificate.clone(),
unsafely_ignore_certificate_errors: workspace_settings
.unsafely_ignore_certificate_errors
.clone(),
},
)?;
for (registry, enabled) in workspace_settings.suggest.imports.hosts.iter() { for (registry, enabled) in workspace_settings.suggest.imports.hosts.iter() {
if *enabled { if *enabled {
lsp_log!("Enabling import suggestions for: {}", registry); lsp_log!("Enabling import suggestions for: {}", registry);

42
cli/lsp/registries.rs
View file

@ -13,10 +13,10 @@ use super::path_to_regex::StringOrVec;
use super::path_to_regex::Token; use super::path_to_regex::Token;
use crate::deno_dir; use crate::deno_dir;
use crate::file_fetcher::get_root_cert_store;
use crate::file_fetcher::CacheSetting; use crate::file_fetcher::CacheSetting;
use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileFetcher;
use crate::http_cache::HttpCache; use crate::http_cache::HttpCache;
use crate::http_util::HttpClient;
use deno_core::anyhow::anyhow; use deno_core::anyhow::anyhow;
use deno_core::error::AnyError; use deno_core::error::AnyError;
@ -36,7 +36,6 @@ use once_cell::sync::Lazy;
use regex::Regex; use regex::Regex;
use std::collections::HashMap; use std::collections::HashMap;
use std::path::Path; use std::path::Path;
use std::path::PathBuf;
use tower_lsp::lsp_types as lsp; use tower_lsp::lsp_types as lsp;
const CONFIG_PATH: &str = "/.well-known/deno-import-intellisense.json"; const CONFIG_PATH: &str = "/.well-known/deno-import-intellisense.json";
@ -409,14 +408,6 @@ enum VariableItems {
List(VariableItemsList), List(VariableItemsList),
} }
#[derive(Debug, Default)]
pub struct ModuleRegistryOptions {
pub maybe_root_path: Option<PathBuf>,
pub maybe_ca_stores: Option<Vec<String>>,
pub maybe_ca_file: Option<String>,
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
}
/// A structure which holds the information about currently configured module /// A structure which holds the information about currently configured module
/// registries and can provide completion information for URLs that match /// registries and can provide completion information for URLs that match
/// one of the enabled registries. /// one of the enabled registries.
@ -433,28 +424,23 @@ impl Default for ModuleRegistry {
// custom root. // custom root.
let dir = deno_dir::DenoDir::new(None).unwrap(); let dir = deno_dir::DenoDir::new(None).unwrap();
let location = dir.root.join("registries"); let location = dir.root.join("registries");
Self::new(&location, ModuleRegistryOptions::default()).unwrap() let http_client = HttpClient::new(None, None).unwrap();
Self::new(&location, http_client).unwrap()
} }
} }
impl ModuleRegistry { impl ModuleRegistry {
pub fn new( pub fn new(
location: &Path, location: &Path,
options: ModuleRegistryOptions, http_client: HttpClient,
) -> Result<Self, AnyError> { ) -> Result<Self, AnyError> {
let http_cache = HttpCache::new(location); let http_cache = HttpCache::new(location);
let root_cert_store = Some(get_root_cert_store(
options.maybe_root_path,
options.maybe_ca_stores,
options.maybe_ca_file,
)?);
let mut file_fetcher = FileFetcher::new( let mut file_fetcher = FileFetcher::new(
http_cache, http_cache,
CacheSetting::RespectHeaders, CacheSetting::RespectHeaders,
true, true,
root_cert_store, http_client,
BlobStore::default(), BlobStore::default(),
options.unsafely_ignore_certificate_errors,
None, None,
)?; )?;
file_fetcher.set_download_log_level(super::logging::lsp_log_level()); file_fetcher.set_download_log_level(super::logging::lsp_log_level());
@ -1262,7 +1248,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let mut module_registry = let mut module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
module_registry module_registry
.enable("http://localhost:4545/") .enable("http://localhost:4545/")
.await .await
@ -1323,7 +1310,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let mut module_registry = let mut module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
module_registry module_registry
.enable("http://localhost:4545/") .enable("http://localhost:4545/")
.await .await
@ -1546,7 +1534,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let mut module_registry = let mut module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
module_registry module_registry
.enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-key-first.json") .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-key-first.json")
.await .await
@ -1616,7 +1605,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let mut module_registry = let mut module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
module_registry module_registry
.enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-complex.json") .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-complex.json")
.await .await
@ -1667,7 +1657,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let module_registry = let module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
let result = module_registry.check_origin("http://localhost:4545").await; let result = module_registry.check_origin("http://localhost:4545").await;
assert!(result.is_ok()); assert!(result.is_ok());
} }
@ -1678,7 +1669,8 @@ mod tests {
let temp_dir = TempDir::new(); let temp_dir = TempDir::new();
let location = temp_dir.path().join("registries"); let location = temp_dir.path().join("registries");
let module_registry = let module_registry =
ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap(); ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
.unwrap();
let result = module_registry.check_origin("https://deno.com").await; let result = module_registry.check_origin("https://deno.com").await;
assert!(result.is_err()); assert!(result.is_err());
let err = result.unwrap_err().to_string(); let err = result.unwrap_err().to_string();

7
cli/npm/cache.rs
View file

@ -10,11 +10,11 @@ use deno_core::anyhow::Context;
use deno_core::error::custom_error; use deno_core::error::custom_error;
use deno_core::error::AnyError; use deno_core::error::AnyError;
use deno_core::url::Url; use deno_core::url::Url;
use deno_runtime::deno_fetch::reqwest;
use crate::deno_dir::DenoDir; use crate::deno_dir::DenoDir;
use crate::file_fetcher::CacheSetting; use crate::file_fetcher::CacheSetting;
use crate::fs_util; use crate::fs_util;
use crate::http_util::HttpClient;
use crate::progress_bar::ProgressBar; use crate::progress_bar::ProgressBar;
use super::registry::NpmPackageVersionDistInfo; use super::registry::NpmPackageVersionDistInfo;
@ -315,6 +315,7 @@ impl ReadonlyNpmCache {
pub struct NpmCache { pub struct NpmCache {
readonly: ReadonlyNpmCache, readonly: ReadonlyNpmCache,
cache_setting: CacheSetting, cache_setting: CacheSetting,
http_client: HttpClient,
progress_bar: ProgressBar, progress_bar: ProgressBar,
} }
@ -322,11 +323,13 @@ impl NpmCache {
pub fn from_deno_dir( pub fn from_deno_dir(
dir: &DenoDir, dir: &DenoDir,
cache_setting: CacheSetting, cache_setting: CacheSetting,
http_client: HttpClient,
progress_bar: ProgressBar, progress_bar: ProgressBar,
) -> Self { ) -> Self {
Self { Self {
readonly: ReadonlyNpmCache::from_deno_dir(dir), readonly: ReadonlyNpmCache::from_deno_dir(dir),
cache_setting, cache_setting,
http_client,
progress_bar, progress_bar,
} }
} }
@ -383,7 +386,7 @@ impl NpmCache {
} }
let _guard = self.progress_bar.update(&dist.tarball); let _guard = self.progress_bar.update(&dist.tarball);
let response = reqwest::get(&dist.tarball).await?; let response = self.http_client.get(&dist.tarball).send().await?;
if response.status() == 404 { if response.status() == 404 {
bail!("Could not find npm package tarball at: {}", dist.tarball); bail!("Could not find npm package tarball at: {}", dist.tarball);

7
cli/npm/registry.rs
View file

@ -18,12 +18,12 @@ use deno_core::serde::Deserialize;
use deno_core::serde_json; use deno_core::serde_json;
use deno_core::url::Url; use deno_core::url::Url;
use deno_runtime::colors; use deno_runtime::colors;
use deno_runtime::deno_fetch::reqwest;
use serde::Serialize; use serde::Serialize;
use crate::file_fetcher::CacheSetting; use crate::file_fetcher::CacheSetting;
use crate::fs_util; use crate::fs_util;
use crate::http_cache::CACHE_PERM; use crate::http_cache::CACHE_PERM;
use crate::http_util::HttpClient;
use crate::progress_bar::ProgressBar; use crate::progress_bar::ProgressBar;
use super::cache::NpmCache; use super::cache::NpmCache;
@ -249,6 +249,7 @@ impl RealNpmRegistryApi {
base_url: Url, base_url: Url,
cache: NpmCache, cache: NpmCache,
cache_setting: CacheSetting, cache_setting: CacheSetting,
http_client: HttpClient,
progress_bar: ProgressBar, progress_bar: ProgressBar,
) -> Self { ) -> Self {
Self(Arc::new(RealNpmRegistryApiInner { Self(Arc::new(RealNpmRegistryApiInner {
@ -256,6 +257,7 @@ impl RealNpmRegistryApi {
cache, cache,
mem_cache: Default::default(), mem_cache: Default::default(),
cache_setting, cache_setting,
http_client,
progress_bar, progress_bar,
})) }))
} }
@ -285,6 +287,7 @@ struct RealNpmRegistryApiInner {
cache: NpmCache, cache: NpmCache,
mem_cache: Mutex<HashMap<String, Option<Arc<NpmPackageInfo>>>>, mem_cache: Mutex<HashMap<String, Option<Arc<NpmPackageInfo>>>>,
cache_setting: CacheSetting, cache_setting: CacheSetting,
http_client: HttpClient,
progress_bar: ProgressBar, progress_bar: ProgressBar,
} }
@ -420,7 +423,7 @@ impl RealNpmRegistryApiInner {
let package_url = self.get_package_url(name); let package_url = self.get_package_url(name);
let _guard = self.progress_bar.update(package_url.as_str()); let _guard = self.progress_bar.update(package_url.as_str());
let response = match reqwest::get(package_url).await { let response = match self.http_client.get(package_url).send().await {
Ok(response) => response, Ok(response) => response,
Err(err) => { Err(err) => {
// attempt to use the local cache // attempt to use the local cache

14
cli/proc_state.rs
View file

@ -19,6 +19,7 @@ use crate::graph_util::graph_lock_or_exit;
use crate::graph_util::GraphData; use crate::graph_util::GraphData;
use crate::graph_util::ModuleEntry; use crate::graph_util::ModuleEntry;
use crate::http_cache; use crate::http_cache;
use crate::http_util::HttpClient;
use crate::lockfile::as_maybe_locker; use crate::lockfile::as_maybe_locker;
use crate::lockfile::Lockfile; use crate::lockfile::Lockfile;
use crate::node; use crate::node;
@ -157,15 +158,18 @@ impl ProcState {
let root_cert_store = cli_options.resolve_root_cert_store()?; let root_cert_store = cli_options.resolve_root_cert_store()?;
let cache_usage = cli_options.cache_setting(); let cache_usage = cli_options.cache_setting();
let progress_bar = ProgressBar::default(); let progress_bar = ProgressBar::default();
let http_client = HttpClient::new(
Some(root_cert_store.clone()),
cli_options
.unsafely_ignore_certificate_errors()
.map(ToOwned::to_owned),
)?;
let file_fetcher = FileFetcher::new( let file_fetcher = FileFetcher::new(
http_cache, http_cache,
cache_usage, cache_usage,
!cli_options.no_remote(), !cli_options.no_remote(),
Some(root_cert_store.clone()), http_client.clone(),
blob_store.clone(), blob_store.clone(),
cli_options
.unsafely_ignore_certificate_errors()
.map(ToOwned::to_owned),
Some(progress_bar.clone()), Some(progress_bar.clone()),
)?; )?;
@ -216,12 +220,14 @@ impl ProcState {
let npm_cache = NpmCache::from_deno_dir( let npm_cache = NpmCache::from_deno_dir(
&dir, &dir,
cli_options.cache_setting(), cli_options.cache_setting(),
http_client.clone(),
progress_bar.clone(), progress_bar.clone(),
); );
let api = RealNpmRegistryApi::new( let api = RealNpmRegistryApi::new(
registry_url, registry_url,
npm_cache.clone(), npm_cache.clone(),
cli_options.cache_setting(), cli_options.cache_setting(),
http_client,
progress_bar.clone(), progress_bar.clone(),
); );
let maybe_lockfile = lockfile.as_ref().cloned(); let maybe_lockfile = lockfile.as_ref().cloned();