2024-01-01 19:58:21 +00:00
|
|
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
2020-09-06 00:34:02 +00:00
|
|
|
|
2022-11-28 22:28:54 +00:00
|
|
|
use crate::args::CacheSetting;
|
2021-02-16 02:50:27 +00:00
|
|
|
use crate::auth_tokens::AuthTokens;
|
2022-11-28 22:28:54 +00:00
|
|
|
use crate::cache::HttpCache;
|
2020-02-18 15:08:18 +00:00
|
|
|
use crate::colors;
|
2021-12-09 11:16:17 +00:00
|
|
|
use crate::http_util::CacheSemantics;
|
2024-06-03 21:17:08 +00:00
|
|
|
use crate::http_util::FetchOnceArgs;
|
|
|
|
use crate::http_util::FetchOnceResult;
|
|
|
|
use crate::http_util::HttpClientProvider;
|
2022-11-28 22:28:54 +00:00
|
|
|
use crate::util::progress_bar::ProgressBar;
|
2021-12-09 11:16:17 +00:00
|
|
|
|
2021-09-07 14:39:32 +00:00
|
|
|
use deno_ast::MediaType;
|
2024-03-15 13:57:24 +00:00
|
|
|
use deno_core::anyhow::bail;
|
2024-02-01 03:15:22 +00:00
|
|
|
use deno_core::anyhow::Context;
|
2020-09-14 16:48:57 +00:00
|
|
|
use deno_core::error::custom_error;
|
|
|
|
use deno_core::error::generic_error;
|
|
|
|
use deno_core::error::uri_error;
|
|
|
|
use deno_core::error::AnyError;
|
2021-07-07 03:48:01 +00:00
|
|
|
use deno_core::parking_lot::Mutex;
|
2022-12-19 19:31:19 +00:00
|
|
|
use deno_core::url::Url;
|
2020-01-05 16:56:18 +00:00
|
|
|
use deno_core::ModuleSpecifier;
|
2024-02-15 19:49:35 +00:00
|
|
|
use deno_graph::source::LoaderChecksum;
|
2024-06-03 21:17:08 +00:00
|
|
|
|
2024-06-07 03:37:53 +00:00
|
|
|
use deno_runtime::deno_permissions::PermissionsContainer;
|
2021-07-05 13:34:37 +00:00
|
|
|
use deno_runtime::deno_web::BlobStore;
|
2021-03-26 16:34:25 +00:00
|
|
|
use log::debug;
|
2024-04-19 01:43:28 +00:00
|
|
|
use std::borrow::Cow;
|
2019-07-31 11:58:41 +00:00
|
|
|
use std::collections::HashMap;
|
2021-02-16 02:50:27 +00:00
|
|
|
use std::env;
|
2019-07-31 11:58:41 +00:00
|
|
|
use std::fs;
|
|
|
|
use std::path::PathBuf;
|
|
|
|
use std::sync::Arc;
|
2021-12-09 11:16:17 +00:00
|
|
|
use std::time::SystemTime;
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2021-04-07 13:22:14 +00:00
|
|
|
pub const SUPPORTED_SCHEMES: [&str; 5] =
|
|
|
|
["data", "blob", "file", "http", "https"];
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
#[derive(Debug, Clone, Eq, PartialEq)]
|
2024-02-01 03:15:22 +00:00
|
|
|
pub struct TextDecodedFile {
|
2020-09-15 14:33:14 +00:00
|
|
|
pub media_type: MediaType,
|
2020-11-06 00:38:21 +00:00
|
|
|
/// The _final_ specifier for the file. The requested specifier and the final
|
|
|
|
/// specifier maybe different for remote files that have been redirected.
|
|
|
|
pub specifier: ModuleSpecifier,
|
2024-02-01 03:15:22 +00:00
|
|
|
/// The source of the file.
|
|
|
|
pub source: Arc<str>,
|
|
|
|
}
|
2021-09-02 15:38:19 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
#[derive(Debug, Clone, Eq, PartialEq)]
|
|
|
|
pub enum FileOrRedirect {
|
|
|
|
File(File),
|
|
|
|
Redirect(ModuleSpecifier),
|
|
|
|
}
|
|
|
|
|
2024-02-01 03:15:22 +00:00
|
|
|
/// A structure representing a source file.
|
|
|
|
#[derive(Debug, Clone, Eq, PartialEq)]
|
|
|
|
pub struct File {
|
|
|
|
/// The _final_ specifier for the file. The requested specifier and the final
|
|
|
|
/// specifier maybe different for remote files that have been redirected.
|
|
|
|
pub specifier: ModuleSpecifier,
|
2021-09-02 15:38:19 +00:00
|
|
|
pub maybe_headers: Option<HashMap<String, String>>,
|
2024-02-01 03:15:22 +00:00
|
|
|
/// The source of the file.
|
|
|
|
pub source: Arc<[u8]>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl File {
|
|
|
|
pub fn resolve_media_type_and_charset(&self) -> (MediaType, Option<&str>) {
|
|
|
|
deno_graph::source::resolve_media_type_and_charset_from_headers(
|
|
|
|
&self.specifier,
|
|
|
|
self.maybe_headers.as_ref(),
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Decodes the source bytes into a string handling any encoding rules
|
|
|
|
/// for local vs remote files and dealing with the charset.
|
|
|
|
pub fn into_text_decoded(self) -> Result<TextDecodedFile, AnyError> {
|
|
|
|
// lots of borrow checker fighting here
|
|
|
|
let (media_type, maybe_charset) =
|
|
|
|
deno_graph::source::resolve_media_type_and_charset_from_headers(
|
|
|
|
&self.specifier,
|
|
|
|
self.maybe_headers.as_ref(),
|
|
|
|
);
|
|
|
|
let specifier = self.specifier;
|
|
|
|
match deno_graph::source::decode_source(
|
|
|
|
&specifier,
|
|
|
|
self.source,
|
|
|
|
maybe_charset,
|
|
|
|
) {
|
|
|
|
Ok(source) => Ok(TextDecodedFile {
|
|
|
|
media_type,
|
|
|
|
specifier,
|
|
|
|
source,
|
|
|
|
}),
|
|
|
|
Err(err) => {
|
|
|
|
Err(err).with_context(|| format!("Failed decoding \"{}\".", specifier))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2021-04-09 01:27:27 +00:00
|
|
|
#[derive(Debug, Clone, Default)]
|
2024-02-10 15:02:31 +00:00
|
|
|
struct MemoryFiles(Arc<Mutex<HashMap<ModuleSpecifier, File>>>);
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2024-02-10 15:02:31 +00:00
|
|
|
impl MemoryFiles {
|
2020-11-06 00:38:21 +00:00
|
|
|
pub fn get(&self, specifier: &ModuleSpecifier) -> Option<File> {
|
2024-02-10 15:02:31 +00:00
|
|
|
self.0.lock().get(specifier).cloned()
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
pub fn insert(&self, specifier: ModuleSpecifier, file: File) -> Option<File> {
|
2024-02-10 15:02:31 +00:00
|
|
|
self.0.lock().insert(specifier, file)
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
2024-05-03 19:52:58 +00:00
|
|
|
|
|
|
|
pub fn clear(&self) {
|
|
|
|
self.0.lock().clear();
|
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
/// Fetch a source file from the local file system.
|
|
|
|
fn fetch_local(specifier: &ModuleSpecifier) -> Result<File, AnyError> {
|
2021-02-17 18:47:18 +00:00
|
|
|
let local = specifier.to_file_path().map_err(|_| {
|
2023-01-27 15:43:16 +00:00
|
|
|
uri_error(format!("Invalid file path.\n Specifier: {specifier}"))
|
2020-11-06 00:38:21 +00:00
|
|
|
})?;
|
2023-07-01 22:52:30 +00:00
|
|
|
let bytes = fs::read(local)?;
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
Ok(File {
|
|
|
|
specifier: specifier.clone(),
|
2021-09-02 15:38:19 +00:00
|
|
|
maybe_headers: None,
|
2024-02-01 03:15:22 +00:00
|
|
|
source: bytes.into(),
|
2020-11-06 00:38:21 +00:00
|
|
|
})
|
|
|
|
}
|
2019-11-22 17:46:57 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
/// Return a validated scheme for a given module specifier.
|
|
|
|
fn get_validated_scheme(
|
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
) -> Result<String, AnyError> {
|
2021-02-17 18:47:18 +00:00
|
|
|
let scheme = specifier.scheme();
|
2020-11-06 00:38:21 +00:00
|
|
|
if !SUPPORTED_SCHEMES.contains(&scheme) {
|
|
|
|
Err(generic_error(format!(
|
2023-01-27 15:43:16 +00:00
|
|
|
"Unsupported scheme \"{scheme}\" for module \"{specifier}\". Supported schemes: {SUPPORTED_SCHEMES:#?}"
|
2020-11-06 00:38:21 +00:00
|
|
|
)))
|
|
|
|
} else {
|
|
|
|
Ok(scheme.to_string())
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
2020-11-06 00:38:21 +00:00
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
pub struct FetchOptions<'a> {
|
|
|
|
pub specifier: &'a ModuleSpecifier,
|
2024-04-19 01:43:28 +00:00
|
|
|
pub permissions: &'a PermissionsContainer,
|
2023-09-07 13:09:16 +00:00
|
|
|
pub maybe_accept: Option<&'a str>,
|
|
|
|
pub maybe_cache_setting: Option<&'a CacheSetting>,
|
2024-04-19 01:43:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub struct FetchNoFollowOptions<'a> {
|
|
|
|
pub fetch_options: FetchOptions<'a>,
|
|
|
|
/// This setting doesn't make sense to provide for `FetchOptions`
|
|
|
|
/// since the required checksum may change for a redirect.
|
|
|
|
pub maybe_checksum: Option<&'a LoaderChecksum>,
|
2023-09-07 13:09:16 +00:00
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
/// A structure for resolving, fetching and caching source files.
|
2024-06-03 21:17:08 +00:00
|
|
|
#[derive(Debug)]
|
2020-11-06 00:38:21 +00:00
|
|
|
pub struct FileFetcher {
|
2021-02-16 02:50:27 +00:00
|
|
|
auth_tokens: AuthTokens,
|
2020-11-06 00:38:21 +00:00
|
|
|
allow_remote: bool,
|
2024-02-10 15:02:31 +00:00
|
|
|
memory_files: MemoryFiles,
|
2020-11-06 00:38:21 +00:00
|
|
|
cache_setting: CacheSetting,
|
2023-08-02 00:49:09 +00:00
|
|
|
http_cache: Arc<dyn HttpCache>,
|
2024-06-03 21:17:08 +00:00
|
|
|
http_client_provider: Arc<HttpClientProvider>,
|
2023-07-01 22:52:30 +00:00
|
|
|
blob_store: Arc<BlobStore>,
|
2021-12-15 18:23:43 +00:00
|
|
|
download_log_level: log::Level,
|
2022-09-09 19:57:39 +00:00
|
|
|
progress_bar: Option<ProgressBar>,
|
2020-11-06 00:38:21 +00:00
|
|
|
}
|
2019-12-03 22:48:53 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
impl FileFetcher {
|
|
|
|
pub fn new(
|
2023-08-02 00:49:09 +00:00
|
|
|
http_cache: Arc<dyn HttpCache>,
|
2020-11-06 00:38:21 +00:00
|
|
|
cache_setting: CacheSetting,
|
|
|
|
allow_remote: bool,
|
2024-06-03 21:17:08 +00:00
|
|
|
http_client_provider: Arc<HttpClientProvider>,
|
2023-07-01 22:52:30 +00:00
|
|
|
blob_store: Arc<BlobStore>,
|
2022-09-09 19:57:39 +00:00
|
|
|
progress_bar: Option<ProgressBar>,
|
2023-01-25 21:51:04 +00:00
|
|
|
) -> Self {
|
|
|
|
Self {
|
2021-11-16 00:59:19 +00:00
|
|
|
auth_tokens: AuthTokens::new(env::var("DENO_AUTH_TOKENS").ok()),
|
2020-11-06 00:38:21 +00:00
|
|
|
allow_remote,
|
2024-02-10 15:02:31 +00:00
|
|
|
memory_files: Default::default(),
|
2020-11-06 00:38:21 +00:00
|
|
|
cache_setting,
|
|
|
|
http_cache,
|
2024-06-03 21:17:08 +00:00
|
|
|
http_client_provider,
|
2021-07-05 13:34:37 +00:00
|
|
|
blob_store,
|
2021-12-15 18:23:43 +00:00
|
|
|
download_log_level: log::Level::Info,
|
2022-09-09 19:57:39 +00:00
|
|
|
progress_bar,
|
2023-01-25 21:51:04 +00:00
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
pub fn cache_setting(&self) -> &CacheSetting {
|
|
|
|
&self.cache_setting
|
|
|
|
}
|
|
|
|
|
2021-12-15 18:23:43 +00:00
|
|
|
/// Sets the log level to use when outputting the download message.
|
|
|
|
pub fn set_download_log_level(&mut self, level: log::Level) {
|
|
|
|
self.download_log_level = level;
|
|
|
|
}
|
|
|
|
|
2019-07-31 11:58:41 +00:00
|
|
|
/// Fetch cached remote file.
|
|
|
|
///
|
|
|
|
/// This is a recursive operation if source file has redirections.
|
2022-03-23 13:54:22 +00:00
|
|
|
pub fn fetch_cached(
|
2020-01-04 10:20:52 +00:00
|
|
|
&self,
|
2020-11-06 00:38:21 +00:00
|
|
|
specifier: &ModuleSpecifier,
|
2020-06-15 21:46:48 +00:00
|
|
|
redirect_limit: i64,
|
2020-11-06 00:38:21 +00:00
|
|
|
) -> Result<Option<File>, AnyError> {
|
2024-04-19 01:43:28 +00:00
|
|
|
let mut specifier = Cow::Borrowed(specifier);
|
|
|
|
for _ in 0..=redirect_limit {
|
|
|
|
match self.fetch_cached_no_follow(&specifier, None)? {
|
|
|
|
Some(FileOrRedirect::File(file)) => {
|
|
|
|
return Ok(Some(file));
|
|
|
|
}
|
|
|
|
Some(FileOrRedirect::Redirect(redirect_specifier)) => {
|
|
|
|
specifier = Cow::Owned(redirect_specifier);
|
|
|
|
}
|
|
|
|
None => {
|
|
|
|
return Ok(None);
|
|
|
|
}
|
|
|
|
}
|
2020-06-15 21:46:48 +00:00
|
|
|
}
|
2024-04-19 01:43:28 +00:00
|
|
|
Err(custom_error("Http", "Too many redirects."))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn fetch_cached_no_follow(
|
|
|
|
&self,
|
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
maybe_checksum: Option<&LoaderChecksum>,
|
|
|
|
) -> Result<Option<FileOrRedirect>, AnyError> {
|
|
|
|
debug!(
|
|
|
|
"FileFetcher::fetch_cached_no_follow - specifier: {}",
|
|
|
|
specifier
|
|
|
|
);
|
2020-06-15 21:46:48 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once
|
2024-02-15 19:49:35 +00:00
|
|
|
let Some(headers) = self.http_cache.read_headers(&cache_key)? else {
|
2023-07-08 20:06:45 +00:00
|
|
|
return Ok(None);
|
2021-02-17 18:47:18 +00:00
|
|
|
};
|
2020-11-06 00:38:21 +00:00
|
|
|
if let Some(redirect_to) = headers.get("location") {
|
|
|
|
let redirect =
|
2021-02-17 18:47:18 +00:00
|
|
|
deno_core::resolve_import(redirect_to, specifier.as_str())?;
|
2024-04-19 01:43:28 +00:00
|
|
|
return Ok(Some(FileOrRedirect::Redirect(redirect)));
|
2020-02-19 13:17:13 +00:00
|
|
|
}
|
2024-05-28 18:58:43 +00:00
|
|
|
let result = self.http_cache.read_file_bytes(
|
2024-02-15 19:49:35 +00:00
|
|
|
&cache_key,
|
|
|
|
maybe_checksum
|
|
|
|
.as_ref()
|
|
|
|
.map(|c| deno_cache_dir::Checksum::new(c.as_str())),
|
|
|
|
deno_cache_dir::GlobalToLocalCopy::Allow,
|
2024-05-28 18:58:43 +00:00
|
|
|
);
|
|
|
|
let bytes = match result {
|
|
|
|
Ok(Some(bytes)) => bytes,
|
|
|
|
Ok(None) => return Ok(None),
|
|
|
|
Err(err) => match err {
|
|
|
|
deno_cache_dir::CacheReadFileError::Io(err) => return Err(err.into()),
|
|
|
|
deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => {
|
|
|
|
// convert to the equivalent deno_graph error so that it
|
|
|
|
// enhances it if this is passed to deno_graph
|
|
|
|
return Err(
|
|
|
|
deno_graph::source::ChecksumIntegrityError {
|
|
|
|
actual: err.actual,
|
|
|
|
expected: err.expected,
|
|
|
|
}
|
|
|
|
.into(),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
},
|
2023-07-08 20:06:45 +00:00
|
|
|
};
|
2020-02-19 13:17:13 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
Ok(Some(FileOrRedirect::File(File {
|
2024-02-01 03:15:22 +00:00
|
|
|
specifier: specifier.clone(),
|
|
|
|
maybe_headers: Some(headers),
|
|
|
|
source: Arc::from(bytes),
|
2024-04-19 01:43:28 +00:00
|
|
|
})))
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2021-01-06 02:22:38 +00:00
|
|
|
/// Convert a data URL into a file, resulting in an error if the URL is
|
|
|
|
/// invalid.
|
|
|
|
fn fetch_data_url(
|
|
|
|
&self,
|
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
) -> Result<File, AnyError> {
|
|
|
|
debug!("FileFetcher::fetch_data_url() - specifier: {}", specifier);
|
2024-02-01 03:15:22 +00:00
|
|
|
let data_url = deno_graph::source::RawDataUrl::parse(specifier)?;
|
|
|
|
let (bytes, headers) = data_url.into_bytes_and_headers();
|
2021-01-06 02:22:38 +00:00
|
|
|
Ok(File {
|
|
|
|
specifier: specifier.clone(),
|
2021-10-10 21:26:22 +00:00
|
|
|
maybe_headers: Some(headers),
|
2024-02-01 03:15:22 +00:00
|
|
|
source: Arc::from(bytes),
|
2021-01-06 02:22:38 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2021-04-07 13:22:14 +00:00
|
|
|
/// Get a blob URL.
|
2021-07-05 13:34:37 +00:00
|
|
|
async fn fetch_blob_url(
|
2021-04-07 13:22:14 +00:00
|
|
|
&self,
|
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
) -> Result<File, AnyError> {
|
|
|
|
debug!("FileFetcher::fetch_blob_url() - specifier: {}", specifier);
|
2023-07-01 22:52:30 +00:00
|
|
|
let blob = self
|
|
|
|
.blob_store
|
|
|
|
.get_object_url(specifier.clone())
|
|
|
|
.ok_or_else(|| {
|
|
|
|
custom_error(
|
|
|
|
"NotFound",
|
|
|
|
format!("Blob URL not found: \"{specifier}\"."),
|
|
|
|
)
|
|
|
|
})?;
|
2021-04-07 13:22:14 +00:00
|
|
|
|
2021-07-05 13:34:37 +00:00
|
|
|
let bytes = blob.read_all().await?;
|
2024-02-01 03:15:22 +00:00
|
|
|
let headers =
|
|
|
|
HashMap::from([("content-type".to_string(), blob.media_type.clone())]);
|
2021-04-07 13:22:14 +00:00
|
|
|
|
|
|
|
Ok(File {
|
|
|
|
specifier: specifier.clone(),
|
2021-10-10 21:26:22 +00:00
|
|
|
maybe_headers: Some(headers),
|
2024-02-01 03:15:22 +00:00
|
|
|
source: Arc::from(bytes),
|
2021-04-07 13:22:14 +00:00
|
|
|
})
|
|
|
|
}
|
2021-12-21 02:40:22 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
async fn fetch_remote_no_follow(
|
2020-01-04 10:20:52 +00:00
|
|
|
&self,
|
2020-11-06 00:38:21 +00:00
|
|
|
specifier: &ModuleSpecifier,
|
2024-04-19 01:43:28 +00:00
|
|
|
maybe_accept: Option<&str>,
|
2023-09-07 13:09:16 +00:00
|
|
|
cache_setting: &CacheSetting,
|
2024-04-19 01:43:28 +00:00
|
|
|
maybe_checksum: Option<&LoaderChecksum>,
|
|
|
|
) -> Result<FileOrRedirect, AnyError> {
|
|
|
|
debug!(
|
|
|
|
"FileFetcher::fetch_remote_no_follow - specifier: {}",
|
|
|
|
specifier
|
|
|
|
);
|
2020-05-11 11:13:27 +00:00
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
if self.should_use_cache(specifier, cache_setting) {
|
2024-04-19 01:43:28 +00:00
|
|
|
if let Some(file_or_redirect) =
|
|
|
|
self.fetch_cached_no_follow(specifier, maybe_checksum)?
|
2024-02-15 19:49:35 +00:00
|
|
|
{
|
2024-04-19 01:43:28 +00:00
|
|
|
return Ok(file_or_redirect);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
if *cache_setting == CacheSetting::Only {
|
2024-04-19 01:43:28 +00:00
|
|
|
return Err(custom_error(
|
2021-12-22 13:25:06 +00:00
|
|
|
"NotCached",
|
2020-11-06 00:38:21 +00:00
|
|
|
format!(
|
2023-01-27 15:43:16 +00:00
|
|
|
"Specifier not found in cache: \"{specifier}\", --cached-only is specified."
|
2020-11-06 00:38:21 +00:00
|
|
|
),
|
2024-04-19 01:43:28 +00:00
|
|
|
));
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2022-12-15 21:23:50 +00:00
|
|
|
let mut maybe_progress_guard = None;
|
2022-09-09 19:57:39 +00:00
|
|
|
if let Some(pb) = self.progress_bar.as_ref() {
|
2022-12-15 21:23:50 +00:00
|
|
|
maybe_progress_guard = Some(pb.update(specifier.as_str()));
|
2022-09-09 19:57:39 +00:00
|
|
|
} else {
|
|
|
|
log::log!(
|
|
|
|
self.download_log_level,
|
|
|
|
"{} {}",
|
|
|
|
colors::green("Download"),
|
|
|
|
specifier
|
|
|
|
);
|
|
|
|
}
|
2020-03-10 12:26:17 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let maybe_etag = self
|
|
|
|
.http_cache
|
2023-08-02 00:49:09 +00:00
|
|
|
.cache_item_key(specifier)
|
2023-07-08 20:06:45 +00:00
|
|
|
.ok()
|
2024-02-15 19:49:35 +00:00
|
|
|
.and_then(|key| self.http_cache.read_headers(&key).ok().flatten())
|
|
|
|
.and_then(|headers| headers.get("etag").cloned());
|
2021-07-30 13:03:41 +00:00
|
|
|
let maybe_auth_token = self.auth_tokens.get(specifier);
|
2023-08-01 08:52:28 +00:00
|
|
|
|
|
|
|
async fn handle_request_or_server_error(
|
|
|
|
retried: &mut bool,
|
|
|
|
specifier: &Url,
|
|
|
|
err_str: String,
|
|
|
|
) -> Result<(), AnyError> {
|
|
|
|
// Retry once, and bail otherwise.
|
|
|
|
if !*retried {
|
|
|
|
*retried = true;
|
|
|
|
log::debug!("Import '{}' failed: {}. Retrying...", specifier, err_str);
|
|
|
|
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
|
|
|
Ok(())
|
|
|
|
} else {
|
|
|
|
Err(generic_error(format!(
|
|
|
|
"Import '{}' failed: {}",
|
|
|
|
specifier, err_str
|
|
|
|
)))
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
let mut maybe_etag = maybe_etag;
|
|
|
|
let mut retried = false; // retry intermittent failures
|
|
|
|
let result = loop {
|
2024-06-03 21:17:08 +00:00
|
|
|
let result = match self
|
|
|
|
.http_client_provider
|
|
|
|
.get_or_create()?
|
|
|
|
.fetch_no_follow(FetchOnceArgs {
|
2024-04-19 01:43:28 +00:00
|
|
|
url: specifier.clone(),
|
|
|
|
maybe_accept: maybe_accept.map(ToOwned::to_owned),
|
|
|
|
maybe_etag: maybe_etag.clone(),
|
|
|
|
maybe_auth_token: maybe_auth_token.clone(),
|
|
|
|
maybe_progress_guard: maybe_progress_guard.as_ref(),
|
2024-06-03 21:17:08 +00:00
|
|
|
})
|
|
|
|
.await?
|
2024-04-19 01:43:28 +00:00
|
|
|
{
|
|
|
|
FetchOnceResult::NotModified => {
|
|
|
|
let file_or_redirect =
|
|
|
|
self.fetch_cached_no_follow(specifier, maybe_checksum)?;
|
|
|
|
match file_or_redirect {
|
|
|
|
Some(file_or_redirect) => Ok(file_or_redirect),
|
|
|
|
None => {
|
|
|
|
// Someone may have deleted the body from the cache since
|
|
|
|
// it's currently stored in a separate file from the headers,
|
|
|
|
// so delete the etag and try again
|
|
|
|
if maybe_etag.is_some() {
|
|
|
|
debug!("Cache body not found. Trying again without etag.");
|
|
|
|
maybe_etag = None;
|
|
|
|
continue;
|
|
|
|
} else {
|
|
|
|
// should never happen
|
|
|
|
bail!("Your deno cache directory is in an unrecoverable state. Please delete it and try again.")
|
2024-03-15 13:57:24 +00:00
|
|
|
}
|
|
|
|
}
|
2023-08-01 08:52:28 +00:00
|
|
|
}
|
2024-04-19 01:43:28 +00:00
|
|
|
}
|
|
|
|
FetchOnceResult::Redirect(redirect_url, headers) => {
|
|
|
|
self.http_cache.set(specifier, headers, &[])?;
|
|
|
|
Ok(FileOrRedirect::Redirect(redirect_url))
|
|
|
|
}
|
|
|
|
FetchOnceResult::Code(bytes, headers) => {
|
|
|
|
self.http_cache.set(specifier, headers.clone(), &bytes)?;
|
|
|
|
if let Some(checksum) = &maybe_checksum {
|
|
|
|
checksum.check_source(&bytes)?;
|
2023-08-01 08:52:28 +00:00
|
|
|
}
|
2024-04-19 01:43:28 +00:00
|
|
|
Ok(FileOrRedirect::File(File {
|
|
|
|
specifier: specifier.clone(),
|
|
|
|
maybe_headers: Some(headers),
|
|
|
|
source: Arc::from(bytes),
|
|
|
|
}))
|
|
|
|
}
|
|
|
|
FetchOnceResult::RequestError(err) => {
|
|
|
|
handle_request_or_server_error(&mut retried, specifier, err).await?;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
FetchOnceResult::ServerError(status) => {
|
|
|
|
handle_request_or_server_error(
|
|
|
|
&mut retried,
|
|
|
|
specifier,
|
|
|
|
status.to_string(),
|
|
|
|
)
|
|
|
|
.await?;
|
|
|
|
continue;
|
|
|
|
}
|
2022-12-15 21:23:50 +00:00
|
|
|
};
|
2024-04-19 01:43:28 +00:00
|
|
|
break result;
|
|
|
|
};
|
2023-08-01 08:52:28 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
drop(maybe_progress_guard);
|
|
|
|
result
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2022-11-28 22:28:54 +00:00
|
|
|
/// Returns if the cache should be used for a given specifier.
|
2023-09-07 13:09:16 +00:00
|
|
|
fn should_use_cache(
|
|
|
|
&self,
|
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
cache_setting: &CacheSetting,
|
|
|
|
) -> bool {
|
|
|
|
match cache_setting {
|
2022-11-28 22:28:54 +00:00
|
|
|
CacheSetting::ReloadAll => false,
|
|
|
|
CacheSetting::Use | CacheSetting::Only => true,
|
|
|
|
CacheSetting::RespectHeaders => {
|
2023-08-02 00:49:09 +00:00
|
|
|
let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else {
|
2023-07-08 20:06:45 +00:00
|
|
|
return false;
|
|
|
|
};
|
2024-02-15 19:49:35 +00:00
|
|
|
let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else {
|
|
|
|
return false;
|
|
|
|
};
|
|
|
|
let Ok(Some(download_time)) =
|
|
|
|
self.http_cache.read_download_time(&cache_key)
|
2023-08-27 04:04:12 +00:00
|
|
|
else {
|
2023-07-08 20:06:45 +00:00
|
|
|
return false;
|
|
|
|
};
|
2024-02-15 19:49:35 +00:00
|
|
|
let cache_semantics =
|
|
|
|
CacheSemantics::new(headers, download_time, SystemTime::now());
|
2023-07-08 20:06:45 +00:00
|
|
|
cache_semantics.should_use()
|
2022-11-28 22:28:54 +00:00
|
|
|
}
|
|
|
|
CacheSetting::ReloadSome(list) => {
|
|
|
|
let mut url = specifier.clone();
|
|
|
|
url.set_fragment(None);
|
2023-01-04 12:20:36 +00:00
|
|
|
if list.iter().any(|x| x == url.as_str()) {
|
2022-11-28 22:28:54 +00:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
url.set_query(None);
|
|
|
|
let mut path = PathBuf::from(url.as_str());
|
|
|
|
loop {
|
|
|
|
if list.contains(&path.to_str().unwrap().to_string()) {
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if !path.pop() {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
/// Fetch a source file and asynchronously return it.
|
|
|
|
pub async fn fetch(
|
|
|
|
&self,
|
|
|
|
specifier: &ModuleSpecifier,
|
2024-04-19 01:43:28 +00:00
|
|
|
permissions: &PermissionsContainer,
|
2020-11-06 00:38:21 +00:00
|
|
|
) -> Result<File, AnyError> {
|
2023-09-07 13:09:16 +00:00
|
|
|
self
|
|
|
|
.fetch_with_options(FetchOptions {
|
|
|
|
specifier,
|
|
|
|
permissions,
|
|
|
|
maybe_accept: None,
|
|
|
|
maybe_cache_setting: None,
|
|
|
|
})
|
|
|
|
.await
|
2021-12-21 02:40:22 +00:00
|
|
|
}
|
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
pub async fn fetch_with_options(
|
2021-12-21 02:40:22 +00:00
|
|
|
&self,
|
2023-09-07 13:09:16 +00:00
|
|
|
options: FetchOptions<'_>,
|
2021-12-21 02:40:22 +00:00
|
|
|
) -> Result<File, AnyError> {
|
2024-04-19 01:43:28 +00:00
|
|
|
self.fetch_with_options_and_max_redirect(options, 10).await
|
|
|
|
}
|
|
|
|
|
|
|
|
async fn fetch_with_options_and_max_redirect(
|
|
|
|
&self,
|
|
|
|
options: FetchOptions<'_>,
|
|
|
|
max_redirect: usize,
|
|
|
|
) -> Result<File, AnyError> {
|
|
|
|
let mut specifier = Cow::Borrowed(options.specifier);
|
|
|
|
for _ in 0..=max_redirect {
|
|
|
|
match self
|
|
|
|
.fetch_no_follow_with_options(FetchNoFollowOptions {
|
|
|
|
fetch_options: FetchOptions {
|
|
|
|
specifier: &specifier,
|
|
|
|
permissions: options.permissions,
|
|
|
|
maybe_accept: options.maybe_accept,
|
|
|
|
maybe_cache_setting: options.maybe_cache_setting,
|
|
|
|
},
|
|
|
|
maybe_checksum: None,
|
|
|
|
})
|
|
|
|
.await?
|
|
|
|
{
|
|
|
|
FileOrRedirect::File(file) => {
|
|
|
|
return Ok(file);
|
|
|
|
}
|
|
|
|
FileOrRedirect::Redirect(redirect_specifier) => {
|
|
|
|
specifier = Cow::Owned(redirect_specifier);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
Err(custom_error("Http", "Too many redirects."))
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Fetches without following redirects.
|
|
|
|
pub async fn fetch_no_follow_with_options(
|
|
|
|
&self,
|
|
|
|
options: FetchNoFollowOptions<'_>,
|
|
|
|
) -> Result<FileOrRedirect, AnyError> {
|
|
|
|
let maybe_checksum = options.maybe_checksum;
|
|
|
|
let options = options.fetch_options;
|
2023-09-07 13:09:16 +00:00
|
|
|
let specifier = options.specifier;
|
2024-04-19 01:43:28 +00:00
|
|
|
// note: this debug output is used by the tests
|
|
|
|
debug!(
|
|
|
|
"FileFetcher::fetch_no_follow_with_options - specifier: {}",
|
|
|
|
specifier
|
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
let scheme = get_validated_scheme(specifier)?;
|
2023-09-07 13:09:16 +00:00
|
|
|
options.permissions.check_specifier(specifier)?;
|
2024-02-10 15:02:31 +00:00
|
|
|
if let Some(file) = self.memory_files.get(specifier) {
|
2024-04-19 01:43:28 +00:00
|
|
|
Ok(FileOrRedirect::File(file))
|
2021-01-06 02:22:38 +00:00
|
|
|
} else if scheme == "file" {
|
|
|
|
// we do not in memory cache files, as this would prevent files on the
|
|
|
|
// disk changing effecting things like workers and dynamic imports.
|
2024-04-19 01:43:28 +00:00
|
|
|
fetch_local(specifier).map(FileOrRedirect::File)
|
2021-01-06 02:22:38 +00:00
|
|
|
} else if scheme == "data" {
|
2024-04-19 01:43:28 +00:00
|
|
|
self.fetch_data_url(specifier).map(FileOrRedirect::File)
|
2021-04-07 13:22:14 +00:00
|
|
|
} else if scheme == "blob" {
|
2024-04-19 01:43:28 +00:00
|
|
|
self
|
|
|
|
.fetch_blob_url(specifier)
|
|
|
|
.await
|
|
|
|
.map(FileOrRedirect::File)
|
2021-01-06 02:22:38 +00:00
|
|
|
} else if !self.allow_remote {
|
|
|
|
Err(custom_error(
|
|
|
|
"NoRemote",
|
2023-01-27 15:43:16 +00:00
|
|
|
format!("A remote specifier was requested: \"{specifier}\", but --no-remote is specified."),
|
2021-01-06 02:22:38 +00:00
|
|
|
))
|
2020-11-06 00:38:21 +00:00
|
|
|
} else {
|
2024-02-10 15:02:31 +00:00
|
|
|
self
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch_remote_no_follow(
|
2021-12-21 02:40:22 +00:00
|
|
|
specifier,
|
2024-04-19 01:43:28 +00:00
|
|
|
options.maybe_accept,
|
2023-09-07 13:09:16 +00:00
|
|
|
options.maybe_cache_setting.unwrap_or(&self.cache_setting),
|
2024-04-19 01:43:28 +00:00
|
|
|
maybe_checksum,
|
2021-12-21 02:40:22 +00:00
|
|
|
)
|
2024-02-10 15:02:31 +00:00
|
|
|
.await
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-11-16 02:19:31 +00:00
|
|
|
/// A synchronous way to retrieve a source file, where if the file has already
|
|
|
|
/// been cached in memory it will be returned, otherwise for local files will
|
|
|
|
/// be read from disk.
|
|
|
|
pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option<File> {
|
2024-02-10 15:02:31 +00:00
|
|
|
let maybe_file = self.memory_files.get(specifier);
|
2020-11-16 02:19:31 +00:00
|
|
|
if maybe_file.is_none() {
|
2021-02-17 18:47:18 +00:00
|
|
|
let is_local = specifier.scheme() == "file";
|
2020-11-16 02:19:31 +00:00
|
|
|
if is_local {
|
|
|
|
if let Ok(file) = fetch_local(specifier) {
|
|
|
|
return Some(file);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
None
|
|
|
|
} else {
|
|
|
|
maybe_file
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-02-10 15:02:31 +00:00
|
|
|
/// Insert a temporary module for the file fetcher.
|
|
|
|
pub fn insert_memory_files(&self, file: File) -> Option<File> {
|
|
|
|
self.memory_files.insert(file.specifier.clone(), file)
|
2019-10-17 14:29:06 +00:00
|
|
|
}
|
2024-05-03 19:52:58 +00:00
|
|
|
|
|
|
|
pub fn clear_memory_files(&self) {
|
|
|
|
self.memory_files.clear();
|
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[cfg(test)]
|
|
|
|
mod tests {
|
2023-08-02 00:49:09 +00:00
|
|
|
use crate::cache::GlobalHttpCache;
|
2023-08-08 14:23:02 +00:00
|
|
|
use crate::cache::RealDenoCacheEnv;
|
2024-06-03 21:17:08 +00:00
|
|
|
use crate::http_util::HttpClientProvider;
|
2022-11-18 22:28:14 +00:00
|
|
|
|
2019-07-31 11:58:41 +00:00
|
|
|
use super::*;
|
2020-11-06 00:38:21 +00:00
|
|
|
use deno_core::error::get_custom_error_class;
|
2021-02-17 18:47:18 +00:00
|
|
|
use deno_core::resolve_url;
|
2021-06-10 13:26:10 +00:00
|
|
|
use deno_runtime::deno_web::Blob;
|
2021-07-05 13:34:37 +00:00
|
|
|
use deno_runtime::deno_web::InMemoryBlobPart;
|
2022-04-01 15:15:37 +00:00
|
|
|
use test_util::TempDir;
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
fn setup(
|
|
|
|
cache_setting: CacheSetting,
|
2022-04-01 15:15:37 +00:00
|
|
|
maybe_temp_dir: Option<TempDir>,
|
|
|
|
) -> (FileFetcher, TempDir) {
|
2021-04-07 13:22:14 +00:00
|
|
|
let (file_fetcher, temp_dir, _) =
|
2021-07-05 13:34:37 +00:00
|
|
|
setup_with_blob_store(cache_setting, maybe_temp_dir);
|
2021-04-07 13:22:14 +00:00
|
|
|
(file_fetcher, temp_dir)
|
|
|
|
}
|
|
|
|
|
2021-07-05 13:34:37 +00:00
|
|
|
fn setup_with_blob_store(
|
2021-04-07 13:22:14 +00:00
|
|
|
cache_setting: CacheSetting,
|
2022-04-01 15:15:37 +00:00
|
|
|
maybe_temp_dir: Option<TempDir>,
|
2023-07-01 22:52:30 +00:00
|
|
|
) -> (FileFetcher, TempDir, Arc<BlobStore>) {
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = maybe_temp_dir.unwrap_or_default();
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2023-07-01 22:52:30 +00:00
|
|
|
let blob_store: Arc<BlobStore> = Default::default();
|
2021-04-07 13:22:14 +00:00
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)),
|
2021-04-07 13:22:14 +00:00
|
|
|
cache_setting,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2021-07-05 13:34:37 +00:00
|
|
|
blob_store.clone(),
|
2021-08-09 14:53:21 +00:00
|
|
|
None,
|
2023-01-25 21:51:04 +00:00
|
|
|
);
|
2021-07-05 13:34:37 +00:00
|
|
|
(file_fetcher, temp_dir, blob_store)
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch(specifier: &ModuleSpecifier) -> (File, FileFetcher) {
|
|
|
|
let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None);
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(specifier, &PermissionsContainer::allow_all())
|
2020-11-06 00:38:21 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
(result.unwrap(), file_fetcher)
|
|
|
|
}
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
async fn test_fetch_options_remote(
|
2020-11-06 00:38:21 +00:00
|
|
|
specifier: &ModuleSpecifier,
|
|
|
|
) -> (File, HashMap<String, String>) {
|
|
|
|
let _http_server_guard = test_util::http_server();
|
|
|
|
let (file_fetcher, _) = setup(CacheSetting::ReloadAll, None);
|
|
|
|
let result: Result<File, AnyError> = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch_with_options_and_max_redirect(
|
|
|
|
FetchOptions {
|
|
|
|
specifier,
|
|
|
|
permissions: &PermissionsContainer::allow_all(),
|
|
|
|
maybe_accept: None,
|
|
|
|
maybe_cache_setting: Some(&file_fetcher.cache_setting),
|
|
|
|
},
|
2023-09-07 13:09:16 +00:00
|
|
|
1,
|
|
|
|
)
|
2020-11-06 00:38:21 +00:00
|
|
|
.await;
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap();
|
2023-07-08 20:06:45 +00:00
|
|
|
(
|
|
|
|
result.unwrap(),
|
2023-08-02 00:49:09 +00:00
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
2023-08-02 00:49:09 +00:00
|
|
|
.unwrap()
|
2024-02-15 19:49:35 +00:00
|
|
|
.unwrap(),
|
2023-07-08 20:06:45 +00:00
|
|
|
)
|
2020-11-06 00:38:21 +00:00
|
|
|
}
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2024-02-01 03:15:22 +00:00
|
|
|
// this test used to test how the file fetcher decoded strings, but
|
|
|
|
// now we're using it as a bit of an integration test with the functionality
|
|
|
|
// in deno_graph
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_remote_encoded(
|
|
|
|
fixture: &str,
|
|
|
|
charset: &str,
|
|
|
|
expected: &str,
|
|
|
|
) {
|
2023-01-27 15:43:16 +00:00
|
|
|
let url_str = format!("http://127.0.0.1:4545/encoding/{fixture}");
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier = resolve_url(&url_str).unwrap();
|
2024-04-19 01:43:28 +00:00
|
|
|
let (file, headers) = test_fetch_options_remote(&specifier).await;
|
2024-02-01 03:15:22 +00:00
|
|
|
let (media_type, maybe_charset) =
|
|
|
|
deno_graph::source::resolve_media_type_and_charset_from_headers(
|
|
|
|
&specifier,
|
|
|
|
Some(&headers),
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
deno_graph::source::decode_source(&specifier, file.source, maybe_charset)
|
|
|
|
.unwrap()
|
|
|
|
.as_ref(),
|
|
|
|
expected
|
|
|
|
);
|
|
|
|
assert_eq!(media_type, MediaType::TypeScript);
|
2020-11-06 00:38:21 +00:00
|
|
|
assert_eq!(
|
|
|
|
headers.get("content-type").unwrap(),
|
2023-01-27 15:43:16 +00:00
|
|
|
&format!("application/typescript;charset={charset}")
|
2020-11-06 00:38:21 +00:00
|
|
|
);
|
|
|
|
}
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_local_encoded(charset: &str, expected: String) {
|
2023-01-27 15:43:16 +00:00
|
|
|
let p = test_util::testdata_path().join(format!("encoding/{charset}.ts"));
|
2023-06-10 15:09:45 +00:00
|
|
|
let specifier = ModuleSpecifier::from_file_path(p).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
let (file, _) = test_fetch(&specifier).await;
|
2024-02-01 03:15:22 +00:00
|
|
|
assert_eq!(
|
|
|
|
deno_graph::source::decode_source(&specifier, file.source, None)
|
|
|
|
.unwrap()
|
|
|
|
.as_ref(),
|
|
|
|
expected
|
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
}
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
#[test]
|
|
|
|
fn test_get_validated_scheme() {
|
|
|
|
let fixtures = vec![
|
|
|
|
("https://deno.land/x/mod.ts", true, "https"),
|
|
|
|
("http://deno.land/x/mod.ts", true, "http"),
|
|
|
|
("file:///a/b/c.ts", true, "file"),
|
|
|
|
("file:///C:/a/b/c.ts", true, "file"),
|
2021-01-06 02:22:38 +00:00
|
|
|
("data:,some%20text", true, "data"),
|
2020-11-06 00:38:21 +00:00
|
|
|
("ftp://a/b/c.ts", false, ""),
|
|
|
|
("mailto:dino@deno.land", false, ""),
|
|
|
|
];
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
for (specifier, is_ok, expected) in fixtures {
|
2023-03-13 23:31:03 +00:00
|
|
|
let specifier = ModuleSpecifier::parse(specifier).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
let actual = get_validated_scheme(&specifier);
|
|
|
|
assert_eq!(actual.is_ok(), is_ok);
|
|
|
|
if is_ok {
|
|
|
|
assert_eq!(actual.unwrap(), expected);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2019-10-17 14:29:06 +00:00
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_insert_cached() {
|
|
|
|
let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None);
|
|
|
|
let local = temp_dir.path().join("a.ts");
|
2023-06-10 15:09:45 +00:00
|
|
|
let specifier = ModuleSpecifier::from_file_path(&local).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
let file = File {
|
2024-02-01 03:15:22 +00:00
|
|
|
source: Arc::from("some source code".as_bytes()),
|
2020-11-06 00:38:21 +00:00
|
|
|
specifier: specifier.clone(),
|
2024-02-01 03:15:22 +00:00
|
|
|
maybe_headers: Some(HashMap::from([(
|
|
|
|
"content-type".to_string(),
|
|
|
|
"application/javascript".to_string(),
|
|
|
|
)])),
|
2020-11-06 00:38:21 +00:00
|
|
|
};
|
2024-02-10 15:02:31 +00:00
|
|
|
file_fetcher.insert_memory_files(file.clone());
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-11-06 00:38:21 +00:00
|
|
|
let result_file = result.unwrap();
|
|
|
|
assert_eq!(result_file, file);
|
|
|
|
}
|
|
|
|
|
2021-01-06 02:22:38 +00:00
|
|
|
#[tokio::test]
|
|
|
|
async fn test_fetch_data_url() {
|
|
|
|
let (file_fetcher, _) = setup(CacheSetting::Use, None);
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier = resolve_url("data:application/typescript;base64,ZXhwb3J0IGNvbnN0IGEgPSAiYSI7CgpleHBvcnQgZW51bSBBIHsKICBBLAogIEIsCiAgQywKfQo=").unwrap();
|
2021-01-06 02:22:38 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-01-06 02:22:38 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2021-01-06 02:22:38 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2021-01-06 02:22:38 +00:00
|
|
|
"export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n"
|
|
|
|
);
|
|
|
|
assert_eq!(file.media_type, MediaType::TypeScript);
|
|
|
|
assert_eq!(file.specifier, specifier);
|
|
|
|
}
|
|
|
|
|
2021-04-07 13:22:14 +00:00
|
|
|
#[tokio::test]
|
|
|
|
async fn test_fetch_blob_url() {
|
2021-07-05 13:34:37 +00:00
|
|
|
let (file_fetcher, _, blob_store) =
|
|
|
|
setup_with_blob_store(CacheSetting::Use, None);
|
|
|
|
|
|
|
|
let bytes =
|
|
|
|
"export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n"
|
|
|
|
.as_bytes()
|
|
|
|
.to_vec();
|
2021-04-07 13:22:14 +00:00
|
|
|
|
2021-07-05 13:34:37 +00:00
|
|
|
let specifier = blob_store.insert_object_url(
|
2021-04-07 13:22:14 +00:00
|
|
|
Blob {
|
|
|
|
media_type: "application/typescript".to_string(),
|
2021-08-14 08:27:27 +00:00
|
|
|
parts: vec![Arc::new(InMemoryBlobPart::from(bytes))],
|
2021-04-07 13:22:14 +00:00
|
|
|
},
|
|
|
|
None,
|
|
|
|
);
|
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-04-07 13:22:14 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2021-04-07 13:22:14 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2021-04-07 13:22:14 +00:00
|
|
|
"export const a = \"a\";\n\nexport enum A {\n A,\n B,\n C,\n}\n"
|
|
|
|
);
|
|
|
|
assert_eq!(file.media_type, MediaType::TypeScript);
|
|
|
|
assert_eq!(file.specifier, specifier);
|
|
|
|
}
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
#[tokio::test]
|
|
|
|
async fn test_fetch_complex() {
|
|
|
|
let _http_server_guard = test_util::http_server();
|
|
|
|
let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None);
|
|
|
|
let (file_fetcher_01, _) = setup(CacheSetting::Use, Some(temp_dir.clone()));
|
|
|
|
let (file_fetcher_02, _) = setup(CacheSetting::Use, Some(temp_dir.clone()));
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier =
|
2023-03-13 23:31:03 +00:00
|
|
|
ModuleSpecifier::parse("http://localhost:4545/subdir/mod2.ts").unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-11-06 00:38:21 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2020-10-07 11:43:44 +00:00
|
|
|
"export { printHello } from \"./print_hello.ts\";\n"
|
2020-02-03 13:53:50 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
assert_eq!(file.media_type, MediaType::TypeScript);
|
2020-02-28 18:15:31 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_item_key =
|
|
|
|
file_fetcher.http_cache.cache_item_key(&specifier).unwrap();
|
2024-02-15 19:49:35 +00:00
|
|
|
let mut headers = HashMap::new();
|
|
|
|
headers.insert("content-type".to_string(), "text/javascript".to_string());
|
2023-07-08 20:06:45 +00:00
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.set(&specifier, headers.clone(), file.source.as_bytes())
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap();
|
2020-02-28 18:15:31 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher_01
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
2020-11-06 00:38:21 +00:00
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2020-02-03 13:53:50 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2020-10-07 11:43:44 +00:00
|
|
|
"export { printHello } from \"./print_hello.ts\";\n"
|
2020-02-03 13:53:50 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
// This validates that when using the cached value, because we modified
|
|
|
|
// the value above.
|
|
|
|
assert_eq!(file.media_type, MediaType::JavaScript);
|
2020-02-19 13:17:13 +00:00
|
|
|
|
2024-02-15 19:49:35 +00:00
|
|
|
let headers2 = file_fetcher_02
|
2023-07-08 20:06:45 +00:00
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_item_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap()
|
2024-02-15 19:49:35 +00:00
|
|
|
.unwrap();
|
|
|
|
assert_eq!(headers2.get("content-type").unwrap(), "text/javascript");
|
|
|
|
headers = HashMap::new();
|
|
|
|
headers.insert("content-type".to_string(), "application/json".to_string());
|
2023-07-08 20:06:45 +00:00
|
|
|
file_fetcher_02
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.set(&specifier, headers.clone(), file.source.as_bytes())
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap();
|
2020-02-28 18:15:31 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher_02
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
2020-11-06 00:38:21 +00:00
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2020-02-03 13:53:50 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2020-10-07 11:43:44 +00:00
|
|
|
"export { printHello } from \"./print_hello.ts\";\n"
|
2020-02-03 13:53:50 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
assert_eq!(file.media_type, MediaType::Json);
|
|
|
|
|
|
|
|
// This creates a totally new instance, simulating another Deno process
|
|
|
|
// invocation and indicates to "cache bust".
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2020-11-06 00:38:21 +00:00
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location,
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2020-11-06 00:38:21 +00:00
|
|
|
CacheSetting::ReloadAll,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-01 22:52:30 +00:00
|
|
|
Default::default(),
|
2021-08-09 14:53:21 +00:00
|
|
|
None,
|
2023-01-25 21:51:04 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2020-02-19 13:17:13 +00:00
|
|
|
assert_eq!(
|
2022-05-20 20:40:55 +00:00
|
|
|
&*file.source,
|
2020-11-06 00:38:21 +00:00
|
|
|
"export { printHello } from \"./print_hello.ts\";\n"
|
2020-02-19 13:17:13 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
assert_eq!(file.media_type, MediaType::TypeScript);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_uses_cache() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier =
|
2021-08-11 14:20:47 +00:00
|
|
|
resolve_url("http://localhost:4545/subdir/mismatch_ext.ts").unwrap();
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let file_modified_01 = {
|
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location.clone(),
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2023-07-08 20:06:45 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-08 20:06:45 +00:00
|
|
|
Default::default(),
|
|
|
|
None,
|
|
|
|
);
|
2019-11-22 17:46:57 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2023-07-08 20:06:45 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key =
|
|
|
|
file_fetcher.http_cache.cache_item_key(&specifier).unwrap();
|
2023-07-08 20:06:45 +00:00
|
|
|
(
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2023-08-02 00:49:09 +00:00
|
|
|
.read_modified_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.read_download_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
)
|
|
|
|
};
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let file_modified_02 = {
|
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location,
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2023-07-08 20:06:45 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-08 20:06:45 +00:00
|
|
|
Default::default(),
|
|
|
|
None,
|
|
|
|
);
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2023-07-08 20:06:45 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2019-11-22 17:46:57 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key =
|
|
|
|
file_fetcher.http_cache.cache_item_key(&specifier).unwrap();
|
2023-07-08 20:06:45 +00:00
|
|
|
(
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2023-08-02 00:49:09 +00:00
|
|
|
.read_modified_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.read_download_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
)
|
|
|
|
};
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
assert_eq!(file_modified_01, file_modified_02);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_redirected() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2020-11-06 00:38:21 +00:00
|
|
|
let (file_fetcher, _) = setup(CacheSetting::Use, None);
|
2021-08-11 14:20:47 +00:00
|
|
|
let specifier =
|
|
|
|
resolve_url("http://localhost:4546/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
|
|
|
let redirected_specifier =
|
|
|
|
resolve_url("http://localhost:4545/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-11-06 00:38:21 +00:00
|
|
|
let file = result.unwrap();
|
|
|
|
assert_eq!(file.specifier, redirected_specifier);
|
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &specifier),
|
|
|
|
"",
|
|
|
|
"redirected files should have empty cached contents"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &specifier),
|
|
|
|
Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()),
|
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &redirected_specifier),
|
|
|
|
"export const redirect = 1;\n"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &redirected_specifier),
|
|
|
|
None,
|
|
|
|
);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_multiple_redirects() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2020-11-06 00:38:21 +00:00
|
|
|
let (file_fetcher, _) = setup(CacheSetting::Use, None);
|
2021-08-11 14:20:47 +00:00
|
|
|
let specifier =
|
|
|
|
resolve_url("http://localhost:4548/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
|
|
|
let redirected_01_specifier =
|
|
|
|
resolve_url("http://localhost:4546/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
|
|
|
let redirected_02_specifier =
|
|
|
|
resolve_url("http://localhost:4545/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-11-06 00:38:21 +00:00
|
|
|
let file = result.unwrap();
|
|
|
|
assert_eq!(file.specifier, redirected_02_specifier);
|
2020-02-03 13:53:50 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &specifier),
|
|
|
|
"",
|
|
|
|
"redirected files should have empty cached contents"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &specifier),
|
|
|
|
Some("http://localhost:4546/subdir/redirects/redirect1.js".to_string()),
|
|
|
|
);
|
2020-02-19 13:17:13 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &redirected_01_specifier),
|
|
|
|
"",
|
|
|
|
"redirected files should have empty cached contents"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &redirected_01_specifier),
|
|
|
|
Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()),
|
|
|
|
);
|
2020-02-03 13:53:50 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &redirected_02_specifier),
|
|
|
|
"export const redirect = 1;\n"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &redirected_02_specifier),
|
|
|
|
None,
|
|
|
|
);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_uses_cache_with_redirects() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier =
|
2021-08-11 14:20:47 +00:00
|
|
|
resolve_url("http://localhost:4548/subdir/mismatch_ext.ts").unwrap();
|
2021-02-17 18:47:18 +00:00
|
|
|
let redirected_specifier =
|
2021-08-11 14:20:47 +00:00
|
|
|
resolve_url("http://localhost:4546/subdir/mismatch_ext.ts").unwrap();
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let metadata_file_modified_01 = {
|
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location.clone(),
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2023-07-08 20:06:45 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-08 20:06:45 +00:00
|
|
|
Default::default(),
|
|
|
|
None,
|
|
|
|
);
|
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2023-07-08 20:06:45 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key = file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.cache_item_key(&redirected_specifier)
|
|
|
|
.unwrap();
|
2023-07-08 20:06:45 +00:00
|
|
|
(
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2023-08-02 00:49:09 +00:00
|
|
|
.read_modified_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.read_download_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
)
|
|
|
|
};
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
let metadata_file_modified_02 = {
|
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location,
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2023-07-08 20:06:45 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-08 20:06:45 +00:00
|
|
|
Default::default(),
|
|
|
|
None,
|
|
|
|
);
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&redirected_specifier, &PermissionsContainer::allow_all())
|
2023-07-08 20:06:45 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-02-03 13:53:50 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
let cache_key = file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.cache_item_key(&redirected_specifier)
|
|
|
|
.unwrap();
|
2023-07-08 20:06:45 +00:00
|
|
|
(
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2023-08-02 00:49:09 +00:00
|
|
|
.read_modified_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
|
|
|
.read_download_time(&cache_key)
|
2023-07-08 20:06:45 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap(),
|
|
|
|
)
|
|
|
|
};
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
assert_eq!(metadata_file_modified_01, metadata_file_modified_02);
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetcher_limits_redirects() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2020-11-06 00:38:21 +00:00
|
|
|
let (file_fetcher, _) = setup(CacheSetting::Use, None);
|
2021-08-11 14:20:47 +00:00
|
|
|
let specifier =
|
|
|
|
resolve_url("http://localhost:4548/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
2019-10-06 19:03:30 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch_with_options_and_max_redirect(
|
|
|
|
FetchOptions {
|
|
|
|
specifier: &specifier,
|
|
|
|
permissions: &PermissionsContainer::allow_all(),
|
|
|
|
maybe_accept: None,
|
|
|
|
maybe_cache_setting: Some(&file_fetcher.cache_setting),
|
|
|
|
},
|
2023-09-07 13:09:16 +00:00
|
|
|
2,
|
|
|
|
)
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch_with_options_and_max_redirect(
|
|
|
|
FetchOptions {
|
|
|
|
specifier: &specifier,
|
|
|
|
permissions: &PermissionsContainer::allow_all(),
|
|
|
|
maybe_accept: None,
|
|
|
|
maybe_cache_setting: Some(&file_fetcher.cache_setting),
|
|
|
|
},
|
2023-09-07 13:09:16 +00:00
|
|
|
1,
|
|
|
|
)
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_err());
|
2020-06-15 21:46:48 +00:00
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
let result = file_fetcher.fetch_cached(&specifier, 2);
|
2020-06-15 21:46:48 +00:00
|
|
|
assert!(result.is_ok());
|
|
|
|
|
2024-04-19 01:43:28 +00:00
|
|
|
let result = file_fetcher.fetch_cached(&specifier, 1);
|
2020-06-15 21:46:48 +00:00
|
|
|
assert!(result.is_err());
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-28 14:04:18 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_same_host_redirect() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2020-11-06 00:38:21 +00:00
|
|
|
let (file_fetcher, _) = setup(CacheSetting::Use, None);
|
2021-02-17 18:47:18 +00:00
|
|
|
let specifier = resolve_url(
|
2021-08-11 14:20:47 +00:00
|
|
|
"http://localhost:4550/REDIRECT/subdir/redirects/redirect1.js",
|
2020-02-28 14:04:18 +00:00
|
|
|
)
|
|
|
|
.unwrap();
|
2021-08-11 14:20:47 +00:00
|
|
|
let redirected_specifier =
|
|
|
|
resolve_url("http://localhost:4550/subdir/redirects/redirect1.js")
|
|
|
|
.unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-28 14:04:18 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-11-06 00:38:21 +00:00
|
|
|
let file = result.unwrap();
|
|
|
|
assert_eq!(file.specifier, redirected_specifier);
|
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &specifier),
|
|
|
|
"",
|
|
|
|
"redirected files should have empty cached contents"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &specifier),
|
|
|
|
Some("/subdir/redirects/redirect1.js".to_string()),
|
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
assert_eq!(
|
|
|
|
get_text_from_cache(&file_fetcher, &redirected_specifier),
|
|
|
|
"export const redirect = 1;\n"
|
|
|
|
);
|
|
|
|
assert_eq!(
|
|
|
|
get_location_header_from_cache(&file_fetcher, &redirected_specifier),
|
|
|
|
None
|
|
|
|
);
|
2020-02-28 14:04:18 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_no_remote() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2020-11-06 00:38:21 +00:00
|
|
|
let file_fetcher = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(
|
|
|
|
location,
|
|
|
|
crate::cache::RealDenoCacheEnv,
|
|
|
|
)),
|
2020-11-06 00:38:21 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
false,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-01 22:52:30 +00:00
|
|
|
Default::default(),
|
2021-08-09 14:53:21 +00:00
|
|
|
None,
|
2023-01-25 21:51:04 +00:00
|
|
|
);
|
2022-09-19 14:32:21 +00:00
|
|
|
let specifier =
|
|
|
|
resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
|
2019-12-03 22:48:53 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_err());
|
2020-11-06 00:38:21 +00:00
|
|
|
let err = result.unwrap_err();
|
|
|
|
assert_eq!(get_custom_error_class(&err), Some("NoRemote"));
|
2022-09-19 14:32:21 +00:00
|
|
|
assert_eq!(err.to_string(), "A remote specifier was requested: \"http://localhost:4545/run/002_hello.ts\", but --no-remote is specified.");
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|
|
|
|
|
2020-02-03 13:53:50 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_cache_only() {
|
2020-08-10 21:31:05 +00:00
|
|
|
let _http_server_guard = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2023-06-10 15:09:45 +00:00
|
|
|
let location = temp_dir.path().join("deps").to_path_buf();
|
2020-11-06 00:38:21 +00:00
|
|
|
let file_fetcher_01 = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(location.clone(), RealDenoCacheEnv)),
|
2020-11-06 00:38:21 +00:00
|
|
|
CacheSetting::Only,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-01 22:52:30 +00:00
|
|
|
Default::default(),
|
2021-08-09 14:53:21 +00:00
|
|
|
None,
|
2023-01-25 21:51:04 +00:00
|
|
|
);
|
2020-11-06 00:38:21 +00:00
|
|
|
let file_fetcher_02 = FileFetcher::new(
|
2023-08-08 14:23:02 +00:00
|
|
|
Arc::new(GlobalHttpCache::new(location, RealDenoCacheEnv)),
|
2020-11-06 00:38:21 +00:00
|
|
|
CacheSetting::Use,
|
|
|
|
true,
|
2024-06-03 21:17:08 +00:00
|
|
|
Arc::new(HttpClientProvider::new(None, None)),
|
2023-07-01 22:52:30 +00:00
|
|
|
Default::default(),
|
2021-08-09 14:53:21 +00:00
|
|
|
None,
|
2023-01-25 21:51:04 +00:00
|
|
|
);
|
2022-09-19 14:32:21 +00:00
|
|
|
let specifier =
|
|
|
|
resolve_url("http://localhost:4545/run/002_hello.ts").unwrap();
|
2019-07-31 11:58:41 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher_01
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
2020-11-06 00:38:21 +00:00
|
|
|
assert!(result.is_err());
|
|
|
|
let err = result.unwrap_err();
|
2022-09-19 14:32:21 +00:00
|
|
|
assert_eq!(err.to_string(), "Specifier not found in cache: \"http://localhost:4545/run/002_hello.ts\", --cached-only is specified.");
|
2023-07-08 20:06:45 +00:00
|
|
|
assert_eq!(get_custom_error_class(&err), Some("NotCached"));
|
2020-06-24 18:12:04 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher_02
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-02-19 13:17:13 +00:00
|
|
|
|
2020-11-06 00:38:21 +00:00
|
|
|
let result = file_fetcher_01
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-02-03 13:53:50 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
|
|
|
|
2020-11-16 02:19:31 +00:00
|
|
|
#[tokio::test]
|
|
|
|
async fn test_fetch_local_bypasses_file_cache() {
|
|
|
|
let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None);
|
|
|
|
let fixture_path = temp_dir.path().join("mod.ts");
|
2023-03-13 23:31:03 +00:00
|
|
|
let specifier = ModuleSpecifier::from_file_path(&fixture_path).unwrap();
|
2021-11-16 00:59:19 +00:00
|
|
|
fs::write(fixture_path.clone(), r#"console.log("hello deno");"#).unwrap();
|
2020-11-16 02:19:31 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-11-16 02:19:31 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
assert_eq!(&*file.source, r#"console.log("hello deno");"#);
|
2020-11-16 02:19:31 +00:00
|
|
|
|
2021-11-16 00:59:19 +00:00
|
|
|
fs::write(fixture_path, r#"console.log("goodbye deno");"#).unwrap();
|
2020-11-16 02:19:31 +00:00
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2020-11-16 02:19:31 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
2024-02-01 03:15:22 +00:00
|
|
|
let file = result.unwrap().into_text_decoded().unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
assert_eq!(&*file.source, r#"console.log("goodbye deno");"#);
|
2020-11-16 02:19:31 +00:00
|
|
|
}
|
|
|
|
|
2021-12-09 11:16:17 +00:00
|
|
|
#[tokio::test]
|
|
|
|
async fn test_respect_cache_revalidates() {
|
|
|
|
let _g = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2021-12-09 11:16:17 +00:00
|
|
|
let (file_fetcher, _) =
|
|
|
|
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
|
|
|
|
let specifier =
|
|
|
|
ModuleSpecifier::parse("http://localhost:4545/dynamic").unwrap();
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-12-09 11:16:17 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
let file = result.unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
let first = file.source;
|
2021-12-09 11:16:17 +00:00
|
|
|
|
|
|
|
let (file_fetcher, _) =
|
|
|
|
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-12-09 11:16:17 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
let file = result.unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
let second = file.source;
|
2021-12-09 11:16:17 +00:00
|
|
|
|
|
|
|
assert_ne!(first, second);
|
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
async fn test_respect_cache_still_fresh() {
|
|
|
|
let _g = test_util::http_server();
|
2022-04-01 15:15:37 +00:00
|
|
|
let temp_dir = TempDir::new();
|
2021-12-09 11:16:17 +00:00
|
|
|
let (file_fetcher, _) =
|
|
|
|
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
|
|
|
|
let specifier =
|
|
|
|
ModuleSpecifier::parse("http://localhost:4545/dynamic_cache").unwrap();
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-12-09 11:16:17 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
let file = result.unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
let first = file.source;
|
2021-12-09 11:16:17 +00:00
|
|
|
|
|
|
|
let (file_fetcher, _) =
|
|
|
|
setup(CacheSetting::RespectHeaders, Some(temp_dir.clone()));
|
|
|
|
let result = file_fetcher
|
2024-04-19 01:43:28 +00:00
|
|
|
.fetch(&specifier, &PermissionsContainer::allow_all())
|
2021-12-09 11:16:17 +00:00
|
|
|
.await;
|
|
|
|
assert!(result.is_ok());
|
|
|
|
let file = result.unwrap();
|
2022-05-20 20:40:55 +00:00
|
|
|
let second = file.source;
|
2021-12-09 11:16:17 +00:00
|
|
|
|
|
|
|
assert_eq!(first, second);
|
|
|
|
}
|
|
|
|
|
2020-08-03 21:39:48 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_local_utf_16be() {
|
2024-02-01 03:15:22 +00:00
|
|
|
let expected =
|
|
|
|
String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_local_encoded("utf-16be", expected).await;
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_local_utf_16le() {
|
2024-02-01 03:15:22 +00:00
|
|
|
let expected =
|
|
|
|
String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_local_encoded("utf-16le", expected).await;
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_local_utf8_with_bom() {
|
2024-02-01 03:15:22 +00:00
|
|
|
let expected =
|
|
|
|
String::from_utf8(b"console.log(\"Hello World\");\x0A".to_vec()).unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_local_encoded("utf-8", expected).await;
|
2020-01-11 10:11:05 +00:00
|
|
|
}
|
2020-01-26 18:59:41 +00:00
|
|
|
|
2020-08-03 21:39:48 +00:00
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_remote_utf16_le() {
|
|
|
|
let expected =
|
2024-02-01 03:15:22 +00:00
|
|
|
std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_remote_encoded("utf-16le.ts", "utf-16le", expected).await;
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_remote_utf16_be() {
|
|
|
|
let expected =
|
2024-02-01 03:15:22 +00:00
|
|
|
std::str::from_utf8(b"console.log(\"Hello World\");\x0A").unwrap();
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_remote_encoded("utf-16be.ts", "utf-16be", expected).await;
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[tokio::test]
|
2020-11-06 00:38:21 +00:00
|
|
|
async fn test_fetch_remote_window_1255() {
|
|
|
|
let expected = "console.log(\"\u{5E9}\u{5DC}\u{5D5}\u{5DD} \
|
2020-08-03 21:39:48 +00:00
|
|
|
\u{5E2}\u{5D5}\u{5DC}\u{5DD}\");\u{A}";
|
2020-11-06 00:38:21 +00:00
|
|
|
test_fetch_remote_encoded("windows-1255", "windows-1255", expected).await;
|
2020-08-03 21:39:48 +00:00
|
|
|
}
|
2022-12-14 13:47:18 +00:00
|
|
|
|
2023-08-02 00:49:09 +00:00
|
|
|
#[track_caller]
|
|
|
|
fn get_text_from_cache(
|
|
|
|
file_fetcher: &FileFetcher,
|
|
|
|
url: &ModuleSpecifier,
|
|
|
|
) -> String {
|
|
|
|
let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap();
|
|
|
|
let bytes = file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_file_bytes(
|
|
|
|
&cache_key,
|
|
|
|
None,
|
|
|
|
deno_cache_dir::GlobalToLocalCopy::Allow,
|
|
|
|
)
|
2023-08-02 00:49:09 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap();
|
|
|
|
String::from_utf8(bytes).unwrap()
|
|
|
|
}
|
|
|
|
|
|
|
|
#[track_caller]
|
|
|
|
fn get_location_header_from_cache(
|
|
|
|
file_fetcher: &FileFetcher,
|
|
|
|
url: &ModuleSpecifier,
|
|
|
|
) -> Option<String> {
|
|
|
|
let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap();
|
|
|
|
file_fetcher
|
|
|
|
.http_cache
|
2024-02-15 19:49:35 +00:00
|
|
|
.read_headers(&cache_key)
|
2023-08-02 00:49:09 +00:00
|
|
|
.unwrap()
|
|
|
|
.unwrap()
|
|
|
|
.remove("location")
|
|
|
|
}
|
2019-07-31 11:58:41 +00:00
|
|
|
}
|