// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::args::CacheSetting; use crate::auth_tokens::AuthTokens; use crate::cache::HttpCache; use crate::colors; use crate::http_util::CacheSemantics; use crate::http_util::FetchOnceArgs; use crate::http_util::FetchOnceResult; use crate::http_util::HttpClientProvider; use crate::util::progress_bar::ProgressBar; use deno_ast::MediaType; use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::generic_error; use deno_core::error::uri_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_core::ModuleSpecifier; use deno_graph::source::LoaderChecksum; use deno_runtime::deno_permissions::PermissionsContainer; use deno_runtime::deno_web::BlobStore; use log::debug; use std::borrow::Cow; use std::collections::HashMap; use std::env; use std::fs; use std::path::PathBuf; use std::sync::Arc; use std::time::SystemTime; pub const SUPPORTED_SCHEMES: [&str; 5] = ["data", "blob", "file", "http", "https"]; #[derive(Debug, Clone, Eq, PartialEq)] pub struct TextDecodedFile { pub media_type: MediaType, /// The _final_ specifier for the file. The requested specifier and the final /// specifier maybe different for remote files that have been redirected. pub specifier: ModuleSpecifier, /// The source of the file. pub source: Arc, } #[derive(Debug, Clone, Eq, PartialEq)] pub enum FileOrRedirect { File(File), Redirect(ModuleSpecifier), } impl FileOrRedirect { fn from_deno_cache_entry( specifier: &ModuleSpecifier, cache_entry: deno_cache_dir::CacheEntry, ) -> Result { if let Some(redirect_to) = cache_entry.metadata.headers.get("location") { let redirect = deno_core::resolve_import(redirect_to, specifier.as_str())?; Ok(FileOrRedirect::Redirect(redirect)) } else { Ok(FileOrRedirect::File(File { specifier: specifier.clone(), maybe_headers: Some(cache_entry.metadata.headers), source: Arc::from(cache_entry.content), })) } } } /// A structure representing a source file. #[derive(Debug, Clone, Eq, PartialEq)] pub struct File { /// The _final_ specifier for the file. The requested specifier and the final /// specifier maybe different for remote files that have been redirected. pub specifier: ModuleSpecifier, pub maybe_headers: Option>, /// The source of the file. pub source: Arc<[u8]>, } impl File { pub fn resolve_media_type_and_charset(&self) -> (MediaType, Option<&str>) { deno_graph::source::resolve_media_type_and_charset_from_headers( &self.specifier, self.maybe_headers.as_ref(), ) } /// Decodes the source bytes into a string handling any encoding rules /// for local vs remote files and dealing with the charset. pub fn into_text_decoded(self) -> Result { // lots of borrow checker fighting here let (media_type, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( &self.specifier, self.maybe_headers.as_ref(), ); let specifier = self.specifier; match deno_graph::source::decode_source( &specifier, self.source, maybe_charset, ) { Ok(source) => Ok(TextDecodedFile { media_type, specifier, source, }), Err(err) => { Err(err).with_context(|| format!("Failed decoding \"{}\".", specifier)) } } } } #[derive(Debug, Clone, Default)] struct MemoryFiles(Arc>>); impl MemoryFiles { pub fn get(&self, specifier: &ModuleSpecifier) -> Option { self.0.lock().get(specifier).cloned() } pub fn insert(&self, specifier: ModuleSpecifier, file: File) -> Option { self.0.lock().insert(specifier, file) } pub fn clear(&self) { self.0.lock().clear(); } } /// Fetch a source file from the local file system. fn fetch_local(specifier: &ModuleSpecifier) -> Result { let local = specifier.to_file_path().map_err(|_| { uri_error(format!("Invalid file path.\n Specifier: {specifier}")) })?; let bytes = fs::read(local)?; Ok(File { specifier: specifier.clone(), maybe_headers: None, source: bytes.into(), }) } /// Return a validated scheme for a given module specifier. fn get_validated_scheme( specifier: &ModuleSpecifier, ) -> Result { let scheme = specifier.scheme(); if !SUPPORTED_SCHEMES.contains(&scheme) { Err(generic_error(format!( "Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes: {SUPPORTED_SCHEMES:#?}" ))) } else { Ok(scheme.to_string()) } } pub struct FetchOptions<'a> { pub specifier: &'a ModuleSpecifier, pub permissions: &'a PermissionsContainer, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, } pub struct FetchNoFollowOptions<'a> { pub fetch_options: FetchOptions<'a>, /// This setting doesn't make sense to provide for `FetchOptions` /// since the required checksum may change for a redirect. pub maybe_checksum: Option<&'a LoaderChecksum>, } /// A structure for resolving, fetching and caching source files. #[derive(Debug)] pub struct FileFetcher { auth_tokens: AuthTokens, allow_remote: bool, memory_files: MemoryFiles, cache_setting: CacheSetting, http_cache: Arc, http_client_provider: Arc, blob_store: Arc, download_log_level: log::Level, progress_bar: Option, } impl FileFetcher { pub fn new( http_cache: Arc, cache_setting: CacheSetting, allow_remote: bool, http_client_provider: Arc, blob_store: Arc, progress_bar: Option, ) -> Self { Self { auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()), allow_remote, memory_files: Default::default(), cache_setting, http_cache, http_client_provider, blob_store, download_log_level: log::Level::Info, progress_bar, } } pub fn cache_setting(&self) -> &CacheSetting { &self.cache_setting } /// Sets the log level to use when outputting the download message. pub fn set_download_log_level(&mut self, level: log::Level) { self.download_log_level = level; } /// Fetch cached remote file. /// /// This is a recursive operation if source file has redirections. pub fn fetch_cached( &self, specifier: &ModuleSpecifier, redirect_limit: i64, ) -> Result, AnyError> { let mut specifier = Cow::Borrowed(specifier); for _ in 0..=redirect_limit { match self.fetch_cached_no_follow(&specifier, None)? { Some(FileOrRedirect::File(file)) => { return Ok(Some(file)); } Some(FileOrRedirect::Redirect(redirect_specifier)) => { specifier = Cow::Owned(redirect_specifier); } None => { return Ok(None); } } } Err(custom_error("Http", "Too many redirects.")) } fn fetch_cached_no_follow( &self, specifier: &ModuleSpecifier, maybe_checksum: Option<&LoaderChecksum>, ) -> Result, AnyError> { debug!( "FileFetcher::fetch_cached_no_follow - specifier: {}", specifier ); let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once let result = self.http_cache.get( &cache_key, maybe_checksum .as_ref() .map(|c| deno_cache_dir::Checksum::new(c.as_str())), ); match result { Ok(Some(cache_data)) => Ok(Some(FileOrRedirect::from_deno_cache_entry( specifier, cache_data, )?)), Ok(None) => Ok(None), Err(err) => match err { deno_cache_dir::CacheReadFileError::Io(err) => Err(err.into()), deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => { // convert to the equivalent deno_graph error so that it // enhances it if this is passed to deno_graph Err( deno_graph::source::ChecksumIntegrityError { actual: err.actual, expected: err.expected, } .into(), ) } }, } } /// Convert a data URL into a file, resulting in an error if the URL is /// invalid. fn fetch_data_url( &self, specifier: &ModuleSpecifier, ) -> Result { debug!("FileFetcher::fetch_data_url() - specifier: {}", specifier); let data_url = deno_graph::source::RawDataUrl::parse(specifier)?; let (bytes, headers) = data_url.into_bytes_and_headers(); Ok(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), }) } /// Get a blob URL. async fn fetch_blob_url( &self, specifier: &ModuleSpecifier, ) -> Result { debug!("FileFetcher::fetch_blob_url() - specifier: {}", specifier); let blob = self .blob_store .get_object_url(specifier.clone()) .ok_or_else(|| { custom_error( "NotFound", format!("Blob URL not found: \"{specifier}\"."), ) })?; let bytes = blob.read_all().await?; let headers = HashMap::from([("content-type".to_string(), blob.media_type.clone())]); Ok(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), }) } async fn fetch_remote_no_follow( &self, specifier: &ModuleSpecifier, maybe_accept: Option<&str>, cache_setting: &CacheSetting, maybe_checksum: Option<&LoaderChecksum>, ) -> Result { debug!( "FileFetcher::fetch_remote_no_follow - specifier: {}", specifier ); if self.should_use_cache(specifier, cache_setting) { if let Some(file_or_redirect) = self.fetch_cached_no_follow(specifier, maybe_checksum)? { return Ok(file_or_redirect); } } if *cache_setting == CacheSetting::Only { return Err(custom_error( "NotCached", format!( "Specifier not found in cache: \"{specifier}\", --cached-only is specified." ), )); } let mut maybe_progress_guard = None; if let Some(pb) = self.progress_bar.as_ref() { maybe_progress_guard = Some(pb.update(specifier.as_str())); } else { log::log!( self.download_log_level, "{} {}", colors::green("Download"), specifier ); } let maybe_etag_cache_entry = self .http_cache .cache_item_key(specifier) .ok() .and_then(|key| { self .http_cache .get( &key, maybe_checksum .as_ref() .map(|c| deno_cache_dir::Checksum::new(c.as_str())), ) .ok() .flatten() }) .and_then(|cache_entry| { cache_entry .metadata .headers .get("etag") .cloned() .map(|etag| (cache_entry, etag)) }); let maybe_auth_token = self.auth_tokens.get(specifier); async fn handle_request_or_server_error( retried: &mut bool, specifier: &Url, err_str: String, ) -> Result<(), AnyError> { // Retry once, and bail otherwise. if !*retried { *retried = true; log::debug!("Import '{}' failed: {}. Retrying...", specifier, err_str); tokio::time::sleep(std::time::Duration::from_millis(50)).await; Ok(()) } else { Err(generic_error(format!( "Import '{}' failed: {}", specifier, err_str ))) } } let mut retried = false; // retry intermittent failures let result = loop { let result = match self .http_client_provider .get_or_create()? .fetch_no_follow(FetchOnceArgs { url: specifier.clone(), maybe_accept: maybe_accept.map(ToOwned::to_owned), maybe_etag: maybe_etag_cache_entry .as_ref() .map(|(_, etag)| etag.clone()), maybe_auth_token: maybe_auth_token.clone(), maybe_progress_guard: maybe_progress_guard.as_ref(), }) .await? { FetchOnceResult::NotModified => { let (cache_entry, _) = maybe_etag_cache_entry.unwrap(); FileOrRedirect::from_deno_cache_entry(specifier, cache_entry) } FetchOnceResult::Redirect(redirect_url, headers) => { self.http_cache.set(specifier, headers, &[])?; Ok(FileOrRedirect::Redirect(redirect_url)) } FetchOnceResult::Code(bytes, headers) => { self.http_cache.set(specifier, headers.clone(), &bytes)?; if let Some(checksum) = &maybe_checksum { checksum.check_source(&bytes)?; } Ok(FileOrRedirect::File(File { specifier: specifier.clone(), maybe_headers: Some(headers), source: Arc::from(bytes), })) } FetchOnceResult::RequestError(err) => { handle_request_or_server_error(&mut retried, specifier, err).await?; continue; } FetchOnceResult::ServerError(status) => { handle_request_or_server_error( &mut retried, specifier, status.to_string(), ) .await?; continue; } }; break result; }; drop(maybe_progress_guard); result } /// Returns if the cache should be used for a given specifier. fn should_use_cache( &self, specifier: &ModuleSpecifier, cache_setting: &CacheSetting, ) -> bool { match cache_setting { CacheSetting::ReloadAll => false, CacheSetting::Use | CacheSetting::Only => true, CacheSetting::RespectHeaders => { let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { return false; }; let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { return false; }; let Ok(Some(download_time)) = self.http_cache.read_download_time(&cache_key) else { return false; }; let cache_semantics = CacheSemantics::new(headers, download_time, SystemTime::now()); cache_semantics.should_use() } CacheSetting::ReloadSome(list) => { let mut url = specifier.clone(); url.set_fragment(None); if list.iter().any(|x| x == url.as_str()) { return false; } url.set_query(None); let mut path = PathBuf::from(url.as_str()); loop { if list.contains(&path.to_str().unwrap().to_string()) { return false; } if !path.pop() { break; } } true } } } /// Fetch a source file and asynchronously return it. pub async fn fetch( &self, specifier: &ModuleSpecifier, permissions: &PermissionsContainer, ) -> Result { self .fetch_with_options(FetchOptions { specifier, permissions, maybe_accept: None, maybe_cache_setting: None, }) .await } pub async fn fetch_with_options( &self, options: FetchOptions<'_>, ) -> Result { self.fetch_with_options_and_max_redirect(options, 10).await } async fn fetch_with_options_and_max_redirect( &self, options: FetchOptions<'_>, max_redirect: usize, ) -> Result { let mut specifier = Cow::Borrowed(options.specifier); for _ in 0..=max_redirect { match self .fetch_no_follow_with_options(FetchNoFollowOptions { fetch_options: FetchOptions { specifier: &specifier, permissions: options.permissions, maybe_accept: options.maybe_accept, maybe_cache_setting: options.maybe_cache_setting, }, maybe_checksum: None, }) .await? { FileOrRedirect::File(file) => { return Ok(file); } FileOrRedirect::Redirect(redirect_specifier) => { specifier = Cow::Owned(redirect_specifier); } } } Err(custom_error("Http", "Too many redirects.")) } /// Fetches without following redirects. pub async fn fetch_no_follow_with_options( &self, options: FetchNoFollowOptions<'_>, ) -> Result { let maybe_checksum = options.maybe_checksum; let options = options.fetch_options; let specifier = options.specifier; // note: this debug output is used by the tests debug!( "FileFetcher::fetch_no_follow_with_options - specifier: {}", specifier ); let scheme = get_validated_scheme(specifier)?; options.permissions.check_specifier(specifier)?; if let Some(file) = self.memory_files.get(specifier) { Ok(FileOrRedirect::File(file)) } else if scheme == "file" { // we do not in memory cache files, as this would prevent files on the // disk changing effecting things like workers and dynamic imports. fetch_local(specifier).map(FileOrRedirect::File) } else if scheme == "data" { self.fetch_data_url(specifier).map(FileOrRedirect::File) } else if scheme == "blob" { self .fetch_blob_url(specifier) .await .map(FileOrRedirect::File) } else if !self.allow_remote { Err(custom_error( "NoRemote", format!("A remote specifier was requested: \"{specifier}\", but --no-remote is specified."), )) } else { self .fetch_remote_no_follow( specifier, options.maybe_accept, options.maybe_cache_setting.unwrap_or(&self.cache_setting), maybe_checksum, ) .await } } /// A synchronous way to retrieve a source file, where if the file has already /// been cached in memory it will be returned, otherwise for local files will /// be read from disk. pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option { let maybe_file = self.memory_files.get(specifier); if maybe_file.is_none() { let is_local = specifier.scheme() == "file"; if is_local { if let Ok(file) = fetch_local(specifier) { return Some(file); } } None } else { maybe_file } } /// Insert a temporary module for the file fetcher. pub fn insert_memory_files(&self, file: File) -> Option { self.memory_files.insert(file.specifier.clone(), file) } pub fn clear_memory_files(&self) { self.memory_files.clear(); } } #[cfg(test)] mod tests { use crate::cache::GlobalHttpCache; use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use super::*; use deno_core::error::get_custom_error_class; use deno_core::resolve_url; use deno_runtime::deno_web::Blob; use deno_runtime::deno_web::InMemoryBlobPart; use test_util::TempDir; fn setup( cache_setting: CacheSetting, maybe_temp_dir: Option, ) -> (FileFetcher, TempDir) { let (file_fetcher, temp_dir, _) = setup_with_blob_store(cache_setting, maybe_temp_dir); (file_fetcher, temp_dir) } fn setup_with_blob_store( cache_setting: CacheSetting, maybe_temp_dir: Option, ) -> (FileFetcher, TempDir, Arc) { let temp_dir = maybe_temp_dir.unwrap_or_default(); let location = temp_dir.path().join("deps").to_path_buf(); let blob_store: Arc = Default::default(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), cache_setting, true, Arc::new(HttpClientProvider::new(None, None)), blob_store.clone(), None, ); (file_fetcher, temp_dir, blob_store) } async fn test_fetch(specifier: &ModuleSpecifier) -> (File, FileFetcher) { let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); let result = file_fetcher .fetch(specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); (result.unwrap(), file_fetcher) } async fn test_fetch_options_remote( specifier: &ModuleSpecifier, ) -> (File, HashMap) { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None); let result: Result = file_fetcher .fetch_with_options_and_max_redirect( FetchOptions { specifier, permissions: &PermissionsContainer::allow_all(), maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, 1, ) .await; let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); ( result.unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), ) } // this test used to test how the file fetcher decoded strings, but // now we're using it as a bit of an integration test with the functionality // in deno_graph async fn test_fetch_remote_encoded( fixture: &str, charset: &str, expected: &str, ) { let url_str = format!("http://127.0.0.1:4545/encoding/{fixture}"); let specifier = resolve_url(&url_str).unwrap(); let (file, headers) = test_fetch_options_remote(&specifier).await; let (media_type, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( &specifier, Some(&headers), ); assert_eq!( deno_graph::source::decode_source(&specifier, file.source, maybe_charset) .unwrap() .as_ref(), expected ); assert_eq!(media_type, MediaType::TypeScript); assert_eq!( headers.get("content-type").unwrap(), &format!("application/typescript;charset={charset}") ); } async fn test_fetch_local_encoded(charset: &str, expected: String) { let p = test_util::testdata_path().join(format!("encoding/{charset}.ts")); let specifier = ModuleSpecifier::from_file_path(p).unwrap(); let (file, _) = test_fetch(&specifier).await; assert_eq!( deno_graph::source::decode_source(&specifier, file.source, None) .unwrap() .as_ref(), expected ); } #[test] fn test_get_validated_scheme() { let fixtures = vec![ ("https://deno.land/x/mod.ts", true, "https"), ("http://deno.land/x/mod.ts", true, "http"), ("file:///a/b/c.ts", true, "file"), ("file:///C:/a/b/c.ts", true, "file"), ("data:,some%20text", true, "data"), ("ftp://a/b/c.ts", false, ""), ("mailto:dino@deno.land", false, ""), ]; for (specifier, is_ok, expected) in fixtures { let specifier = ModuleSpecifier::parse(specifier).unwrap(); let actual = get_validated_scheme(&specifier); assert_eq!(actual.is_ok(), is_ok); if is_ok { assert_eq!(actual.unwrap(), expected); } } } #[tokio::test] async fn test_insert_cached() { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let local = temp_dir.path().join("a.ts"); let specifier = ModuleSpecifier::from_file_path(&local).unwrap(); let file = File { source: Arc::from("some source code".as_bytes()), specifier: specifier.clone(), maybe_headers: Some(HashMap::from([( "content-type".to_string(), "application/javascript".to_string(), )])), }; file_fetcher.insert_memory_files(file.clone()); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let result_file = result.unwrap(); assert_eq!(result_file, file); } #[tokio::test] async fn test_fetch_data_url() { let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" ); assert_eq!(file.media_type, MediaType::TypeScript); assert_eq!(file.specifier, specifier); } #[tokio::test] async fn test_fetch_blob_url() { let (file_fetcher, _, blob_store) = setup_with_blob_store(CacheSetting::Use, None); let bytes = "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" .as_bytes() .to_vec(); let specifier = blob_store.insert_object_url( Blob { media_type: "application/typescript".to_string(), parts: vec![Arc::new(InMemoryBlobPart::from(bytes))], }, None, ); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n" ); assert_eq!(file.media_type, MediaType::TypeScript); assert_eq!(file.specifier, specifier); } #[tokio::test] async fn test_fetch_complex() { let _http_server_guard = test_util::http_server(); let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let (file_fetcher_01, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); let (file_fetcher_02, _) = setup(CacheSetting::Use, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/subdir/mod2.ts").unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::TypeScript); let cache_item_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); let mut headers = HashMap::new(); headers.insert("content-type".to_string(), "text/javascript".to_string()); file_fetcher .http_cache .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_01 .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); // This validates that when using the cached value, because we modified // the value above. assert_eq!(file.media_type, MediaType::JavaScript); let headers2 = file_fetcher_02 .http_cache .read_headers(&cache_item_key) .unwrap() .unwrap(); assert_eq!(headers2.get("content-type").unwrap(), "text/javascript"); headers = HashMap::new(); headers.insert("content-type".to_string(), "application/json".to_string()); file_fetcher_02 .http_cache .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_02 .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::Json); // This creates a totally new instance, simulating another Deno process // invocation and indicates to "cache bust". let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::ReloadAll, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!( &*file.source, "export { printHello } from \"./print_hello.ts\";\n" ); assert_eq!(file.media_type, MediaType::TypeScript); } #[tokio::test] async fn test_fetch_uses_cache() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let specifier = resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap(); let file_modified_01 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location.clone(), crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let cache_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) }; let file_modified_02 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let cache_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) }; assert_eq!(file_modified_01, file_modified_02); } #[tokio::test] async fn test_fetch_redirected() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4546/subdir/redirects/redirect1.js") .unwrap(); let redirected_specifier = resolve_url("http://localhost:4545/subdir/redirects/redirect1.js") .unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier); assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( get_text_from_cache(&file_fetcher, &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_specifier), None, ); } #[tokio::test] async fn test_fetch_multiple_redirects() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4548/subdir/redirects/redirect1.js") .unwrap(); let redirected_01_specifier = resolve_url("http://localhost:4546/subdir/redirects/redirect1.js") .unwrap(); let redirected_02_specifier = resolve_url("http://localhost:4545/subdir/redirects/redirect1.js") .unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_02_specifier); assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("http://localhost:4546/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( get_text_from_cache(&file_fetcher, &redirected_01_specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_01_specifier), Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( get_text_from_cache(&file_fetcher, &redirected_02_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_02_specifier), None, ); } #[tokio::test] async fn test_fetch_uses_cache_with_redirects() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let specifier = resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap(); let redirected_specifier = resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap(); let metadata_file_modified_01 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location.clone(), crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let cache_key = file_fetcher .http_cache .cache_item_key(&redirected_specifier) .unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) }; let metadata_file_modified_02 = { let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let result = file_fetcher .fetch(&redirected_specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let cache_key = file_fetcher .http_cache .cache_item_key(&redirected_specifier) .unwrap(); ( file_fetcher .http_cache .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap(), file_fetcher .http_cache .read_download_time(&cache_key) .unwrap() .unwrap(), ) }; assert_eq!(metadata_file_modified_01, metadata_file_modified_02); } #[tokio::test] async fn test_fetcher_limits_redirects() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url("http://localhost:4548/subdir/redirects/redirect1.js") .unwrap(); let result = file_fetcher .fetch_with_options_and_max_redirect( FetchOptions { specifier: &specifier, permissions: &PermissionsContainer::allow_all(), maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, 2, ) .await; assert!(result.is_ok()); let result = file_fetcher .fetch_with_options_and_max_redirect( FetchOptions { specifier: &specifier, permissions: &PermissionsContainer::allow_all(), maybe_accept: None, maybe_cache_setting: Some(&file_fetcher.cache_setting), }, 1, ) .await; assert!(result.is_err()); let result = file_fetcher.fetch_cached(&specifier, 2); assert!(result.is_ok()); let result = file_fetcher.fetch_cached(&specifier, 1); assert!(result.is_err()); } #[tokio::test] async fn test_fetch_same_host_redirect() { let _http_server_guard = test_util::http_server(); let (file_fetcher, _) = setup(CacheSetting::Use, None); let specifier = resolve_url( "http://localhost:4550/REDIRECT/subdir/redirects/redirect1.js", ) .unwrap(); let redirected_specifier = resolve_url("http://localhost:4550/subdir/redirects/redirect1.js") .unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier); assert_eq!( get_text_from_cache(&file_fetcher, &specifier), "", "redirected files should have empty cached contents" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &specifier), Some("/subdir/redirects/redirect1.js".to_string()), ); assert_eq!( get_text_from_cache(&file_fetcher, &redirected_specifier), "export const redirect = 1;\n" ); assert_eq!( get_location_header_from_cache(&file_fetcher, &redirected_specifier), None ); } #[tokio::test] async fn test_fetch_no_remote() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( Arc::new(GlobalHttpCache::new( location, crate::cache::RealDenoCacheEnv, )), CacheSetting::Use, false, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(get_custom_error_class(&err), Some("NoRemote")); assert_eq!(err.to_string(), "A remote specifier was requested: \"http://localhost:4545/run/002_hello.ts\", but --no-remote is specified."); } #[tokio::test] async fn test_fetch_cache_only() { let _http_server_guard = test_util::http_server(); let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher_01 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)), CacheSetting::Only, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let file_fetcher_02 = FileFetcher::new( Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)), CacheSetting::Use, true, Arc::new(HttpClientProvider::new(None, None)), Default::default(), None, ); let specifier = resolve_url("http://localhost:4545/run/002_hello.ts").unwrap(); let result = file_fetcher_01 .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_err()); let err = result.unwrap_err(); assert_eq!(err.to_string(), "Specifier not found in cache: \"http://localhost:4545/run/002_hello.ts\", --cached-only is specified."); assert_eq!(get_custom_error_class(&err), Some("NotCached")); let result = file_fetcher_02 .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let result = file_fetcher_01 .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); } #[tokio::test] async fn test_fetch_local_bypasses_file_cache() { let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); let fixture_path = temp_dir.path().join("mod.ts"); let specifier = ModuleSpecifier::from_file_path(&fixture_path).unwrap(); fs::write(fixture_path.clone(), r#"console.log("hello deno");"#).unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!(&*file.source, r#"console.log("hello deno");"#); fs::write(fixture_path, r#"console.log("goodbye deno");"#).unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap().into_text_decoded().unwrap(); assert_eq!(&*file.source, r#"console.log("goodbye deno");"#); } #[tokio::test] async fn test_respect_cache_revalidates() { let _g = test_util::http_server(); let temp_dir = TempDir::new(); let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/dynamic").unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let first = file.source; let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let second = file.source; assert_ne!(first, second); } #[tokio::test] async fn test_respect_cache_still_fresh() { let _g = test_util::http_server(); let temp_dir = TempDir::new(); let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let specifier = ModuleSpecifier::parse("http://localhost:4545/dynamic_cache").unwrap(); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let first = file.source; let (file_fetcher, _) = setup(CacheSetting::RespectHeaders, Some(temp_dir.clone())); let result = file_fetcher .fetch(&specifier, &PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); let file = result.unwrap(); let second = file.source; assert_eq!(first, second); } #[tokio::test] async fn test_fetch_local_utf_16be() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-16be", expected).await; } #[tokio::test] async fn test_fetch_local_utf_16le() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-16le", expected).await; } #[tokio::test] async fn test_fetch_local_utf8_with_bom() { let expected = String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap(); test_fetch_local_encoded("utf-8", expected).await; } #[tokio::test] async fn test_fetch_remote_utf16_le() { let expected = std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap(); test_fetch_remote_encoded("utf-16le.ts", "utf-16le", expected).await; } #[tokio::test] async fn test_fetch_remote_utf16_be() { let expected = std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap(); test_fetch_remote_encoded("utf-16be.ts", "utf-16be", expected).await; } #[tokio::test] async fn test_fetch_remote_window_1255() { let expected = "console.log(\"\u{5E9}\u{5DC}\u{5D5}\u{5DD} \ \u{5E2}\u{5D5}\u{5DC}\u{5DD}\");\u{A}"; test_fetch_remote_encoded("windows-1255", "windows-1255", expected).await; } #[track_caller] fn get_text_from_cache( file_fetcher: &FileFetcher, url: &ModuleSpecifier, ) -> String { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); let bytes = file_fetcher .http_cache .get(&cache_key, None) .unwrap() .unwrap() .content; String::from_utf8(bytes).unwrap() } #[track_caller] fn get_location_header_from_cache( file_fetcher: &FileFetcher, url: &ModuleSpecifier, ) -> Option { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); file_fetcher .http_cache .read_headers(&cache_key) .unwrap() .unwrap() .remove("location") } }