refactor(cli): refactor file_fetcher (#8245)

cli/main.rs

@@ -56,8 +56,8 @@ mod worker;
 
 use crate::coverage::CoverageCollector;
 use crate::coverage::PrettyCoverageReporter;
-use crate::file_fetcher::SourceFile;
-use crate::file_fetcher::SourceFileFetcher;
+use crate::file_fetcher::File;
+use crate::file_fetcher::FileFetcher;
 use crate::fs as deno_fs;
 use crate::media_type::MediaType;
 use crate::permissions::Permissions;
@@ -117,7 +117,7 @@ fn print_cache_info(
   json: bool,
 ) -> Result<(), AnyError> {
   let deno_dir = &state.dir.root;
-  let modules_cache = &state.file_fetcher.http_cache.location;
+  let modules_cache = &state.file_fetcher.get_http_cache_location();
   let typescript_cache = &state.dir.gen_cache.location;
   if json {
     let output = json!({
@@ -286,22 +286,21 @@ async fn eval_command(
   }
   .into_bytes();
 
-  let source_file = SourceFile {
-    filename: main_module_url.to_file_path().unwrap(),
-    url: main_module_url,
-    types_header: None,
+  let file = File {
+    local: main_module_url.to_file_path().unwrap(),
+    maybe_types: None,
     media_type: if as_typescript {
       MediaType::TypeScript
     } else {
       MediaType::JavaScript
     },
-    source_code: String::from_utf8(source_code)?,
+    source: String::from_utf8(source_code)?,
+    specifier: ModuleSpecifier::from(main_module_url),
   };
+
   // Save our fake file into file fetcher cache
   // to allow module access by TS compiler.
-  program_state
-    .file_fetcher
-    .save_source_file_in_cache(&main_module, source_file);
+  program_state.file_fetcher.insert_cached(file);
   debug!("main_module {}", &main_module);
   worker.execute_module(&main_module).await?;
   worker.execute("window.dispatchEvent(new Event('load'))")?;
@@ -397,7 +396,7 @@ async fn bundle_command(
 }
 
 struct DocLoader {
-  fetcher: SourceFileFetcher,
+  fetcher: FileFetcher,
   maybe_import_map: Option<ImportMap>,
 }
 
@@ -435,7 +434,7 @@ impl DocFileLoader for DocLoader {
       .expect("Expected valid specifier");
     async move {
       let source_file = fetcher
-        .fetch_source_file(&specifier, None, Permissions::allow_all())
+        .fetch(&specifier, &Permissions::allow_all())
         .await
         .map_err(|e| {
           doc::DocError::Io(std::io::Error::new(
@@ -443,7 +442,7 @@ impl DocFileLoader for DocLoader {
             e.to_string(),
           ))
         })?;
-      Ok(source_file.source_code)
+      Ok(source_file.source)
     }
     .boxed_local()
   }
@@ -541,18 +540,16 @@ async fn run_from_stdin(flags: Flags) -> Result<(), AnyError> {
   std::io::stdin().read_to_end(&mut source)?;
   let main_module_url = main_module.as_url().to_owned();
   // Create a dummy source file.
-  let source_file = SourceFile {
-    filename: main_module_url.to_file_path().unwrap(),
-    url: main_module_url,
-    types_header: None,
+  let source_file = File {
+    local: main_module_url.to_file_path().unwrap(),
+    maybe_types: None,
     media_type: MediaType::TypeScript,
-    source_code: String::from_utf8(source)?,
+    source: String::from_utf8(source)?,
+    specifier: main_module.clone(),
   };
   // Save our fake file into file fetcher cache
   // to allow module access by TS compiler
-  program_state
-    .file_fetcher
-    .save_source_file_in_cache(&main_module, source_file);
+  program_state.file_fetcher.insert_cached(source_file);
 
   debug!("main_module {}", main_module);
   worker.execute_module(&main_module).await?;
@@ -671,18 +668,16 @@ async fn test_command(
   let mut worker =
     MainWorker::new(&program_state, main_module.clone(), permissions);
   // Create a dummy source file.
-  let source_file = SourceFile {
-    filename: test_file_url.to_file_path().unwrap(),
-    url: test_file_url.clone(),
-    types_header: None,
+  let source_file = File {
+    local: test_file_url.to_file_path().unwrap(),
+    maybe_types: None,
     media_type: MediaType::TypeScript,
-    source_code: test_file.clone(),
+    source: test_file.clone(),
+    specifier: ModuleSpecifier::from(test_file_url.clone()),
   };
   // Save our fake file into file fetcher cache
   // to allow module access by TS compiler
-  program_state
-    .file_fetcher
-    .save_source_file_in_cache(&main_module, source_file);
+  program_state.file_fetcher.insert_cached(source_file);
 
   let mut maybe_coverage_collector = if flags.coverage {
     let session = worker.create_inspector_session();

cli/media_type.rs

@@ -236,6 +236,23 @@ mod tests {
     assert_eq!(MediaType::from(Path::new("foo/bar")), MediaType::Unknown);
   }
 
+  #[test]
+  fn test_from_specifier() {
+    let fixtures = vec![
+      ("file:///a/b/c.ts", MediaType::TypeScript),
+      ("file:///a/b/c.js", MediaType::JavaScript),
+      ("file:///a/b/c.txt", MediaType::Unknown),
+      ("https://deno.land/x/mod.ts", MediaType::TypeScript),
+      ("https://deno.land/x/mod.js", MediaType::JavaScript),
+      ("https://deno.land/x/mod.txt", MediaType::Unknown),
+    ];
+
+    for (specifier, expected) in fixtures {
+      let actual = ModuleSpecifier::resolve_url_or_path(specifier).unwrap();
+      assert_eq!(MediaType::from(&actual), expected);
+    }
+  }
+
   #[test]
   fn test_serialization() {
     assert_eq!(json!(MediaType::JavaScript), json!(0));

cli/program_state.rs

@@ -1,7 +1,8 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 use crate::deno_dir;
-use crate::file_fetcher::SourceFileFetcher;
+use crate::file_fetcher::CacheSetting;
+use crate::file_fetcher::FileFetcher;
 use crate::flags;
 use crate::http_cache;
 use crate::import_map::ImportMap;
@@ -47,7 +48,7 @@ pub struct ProgramState {
   /// Flags parsed from `argv` contents.
   pub flags: flags::Flags,
   pub dir: deno_dir::DenoDir,
-  pub file_fetcher: SourceFileFetcher,
+  pub file_fetcher: FileFetcher,
   pub lockfile: Option<Arc<Mutex<Lockfile>>>,
   pub maybe_import_map: Option<ImportMap>,
   pub maybe_inspector_server: Option<Arc<InspectorServer>>,
@@ -61,12 +62,20 @@ impl ProgramState {
     let http_cache = http_cache::HttpCache::new(&deps_cache_location);
     let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok());
 
-    let file_fetcher = SourceFileFetcher::new(
+    let cache_usage = if flags.cached_only {
+      CacheSetting::Only
+    } else if !flags.cache_blocklist.is_empty() {
+      CacheSetting::ReloadSome(flags.cache_blocklist.clone())
+    } else if flags.reload {
+      CacheSetting::ReloadAll
+    } else {
+      CacheSetting::Use
+    };
+
+    let file_fetcher = FileFetcher::new(
       http_cache,
-      !flags.reload,
-      flags.cache_blocklist.clone(),
-      flags.no_remote,
-      flags.cached_only,
+      cache_usage,
+      !flags.no_remote,
       ca_file.as_deref(),
     )?;
 
@@ -175,16 +184,20 @@ impl ProgramState {
     module_specifier: ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<CompiledModule, AnyError> {
+    // TODO(@kitsonk) this really needs to be avoided and refactored out, as we
+    // really should just be getting this from the module graph.
     let out = self
       .file_fetcher
-      .fetch_cached_source_file(&module_specifier, Permissions::allow_all())
+      .get_cached(&module_specifier)
       .expect("Cached source file doesn't exist");
 
-    let url = out.url.clone();
-    let compiled_module = if let Some((code, _)) = self.get_emit(&url) {
+    let specifier = out.specifier.clone();
+    let compiled_module = if let Some((code, _)) =
+      self.get_emit(&specifier.as_url())
+    {
       CompiledModule {
         code: String::from_utf8(code).unwrap(),
-        name: out.url.to_string(),
+        name: specifier.as_url().to_string(),
       }
     // We expect a compiled source for any non-JavaScript files, except for
     // local files that have an unknown media type and no referrer (root modules
@@ -192,7 +205,7 @@ impl ProgramState {
     } else if out.media_type != MediaType::JavaScript
       && !(out.media_type == MediaType::Unknown
         && maybe_referrer.is_none()
-        && url.scheme() == "file")
+        && specifier.as_url().scheme() == "file")
     {
       let message = if let Some(referrer) = maybe_referrer {
         format!("Compiled module not found \"{}\"\n  From: {}\n    If the source module contains only types, use `import type` and `export type` to import it instead.", module_specifier, referrer)
@@ -202,12 +215,12 @@ impl ProgramState {
       info!("{}: {}", crate::colors::yellow("warning"), message);
       CompiledModule {
         code: "".to_string(),
-        name: out.url.to_string(),
+        name: specifier.as_url().to_string(),
       }
     } else {
       CompiledModule {
-        code: out.source_code,
-        name: out.url.to_string(),
+        code: out.source,
+        name: specifier.as_url().to_string(),
       }
     };
 
@@ -310,16 +323,13 @@ impl SourceMapGetter for ProgramState {
     line_number: usize,
   ) -> Option<String> {
     if let Ok(specifier) = ModuleSpecifier::resolve_url(file_name) {
-      self
-        .file_fetcher
-        .fetch_cached_source_file(&specifier, Permissions::allow_all())
-        .map(|out| {
-          // Do NOT use .lines(): it skips the terminating empty line.
-          // (due to internally using .split_terminator() instead of .split())
-          let lines: Vec<&str> = out.source_code.split('\n').collect();
-          assert!(lines.len() > line_number);
-          lines[line_number].to_string()
-        })
+      self.file_fetcher.get_cached(&specifier).map(|out| {
+        // Do NOT use .lines(): it skips the terminating empty line.
+        // (due to internally using .split_terminator() instead of .split())
+        let lines: Vec<&str> = out.source.split('\n').collect();
+        assert!(lines.len() > line_number);
+        lines[line_number].to_string()
+      })
     } else {
       None
     }

cli/specifier_handler.rs

@@ -3,7 +3,7 @@
 use crate::ast::Location;
 use crate::deno_dir::DenoDir;
 use crate::disk_cache::DiskCache;
-use crate::file_fetcher::SourceFileFetcher;
+use crate::file_fetcher::FileFetcher;
 use crate::media_type::MediaType;
 use crate::permissions::Permissions;
 use crate::program_state::ProgramState;
@@ -220,7 +220,7 @@ pub struct FetchHandler {
   /// dynamic imports.
   runtime_permissions: Permissions,
   /// A clone of the `program_state` file fetcher.
-  file_fetcher: SourceFileFetcher,
+  file_fetcher: FileFetcher,
 }
 
 impl FetchHandler {
@@ -258,20 +258,35 @@ impl SpecifierHandler for FetchHandler {
     };
     let file_fetcher = self.file_fetcher.clone();
     let disk_cache = self.disk_cache.clone();
-    let maybe_referrer: Option<ModuleSpecifier> =
-      if let Some(location) = &maybe_location {
-        Some(location.clone().into())
-      } else {
-        None
-      };
 
     async move {
       let source_file = file_fetcher
-        .fetch_source_file(&requested_specifier, maybe_referrer, permissions)
+        .fetch(&requested_specifier, &permissions)
         .await
         .map_err(|err| {
+          let err = if let Some(e) = err.downcast_ref::<std::io::Error>() {
+            if e.kind() == std::io::ErrorKind::NotFound {
+              let message = if let Some(location) = &maybe_location {
+                format!(
+                  "Cannot resolve module \"{}\" from \"{}\".",
+                  requested_specifier, location.filename
+                )
+              } else {
+                format!("Cannot resolve module \"{}\".", requested_specifier)
+              };
+              custom_error("NotFound", message)
+            } else {
+              err
+            }
+          } else {
+            err
+          };
           if let Some(location) = maybe_location {
-            if !is_dynamic {
+            // Injected modules (like test and eval) come with locations, but
+            // they are confusing to the user to print out the location because
+            // they cannot actually get to the source code that is quoted, as
+            // it only exists in the runtime memory of Deno.
+            if !location.filename.contains("$deno$") {
               HandlerError::FetchErrorWithLocation(err.to_string(), location)
                 .into()
             } else {
@@ -281,9 +296,9 @@ impl SpecifierHandler for FetchHandler {
             err
           }
         })?;
-      let url = source_file.url.clone();
+      let url = source_file.specifier.as_url();
       let is_remote = url.scheme() != "file";
-      let filename = disk_cache.get_cache_filename_with_extension(&url, "meta");
+      let filename = disk_cache.get_cache_filename_with_extension(url, "meta");
       let maybe_version = if let Ok(bytes) = disk_cache.get(&filename) {
         if let Ok(compiled_file_metadata) =
           CompiledFileMetadata::from_bytes(&bytes)
@@ -313,20 +328,19 @@ impl SpecifierHandler for FetchHandler {
         maybe_emit_path =
           Some((disk_cache.location.join(emit_path), maybe_map_path));
       };
-      let specifier = ModuleSpecifier::from(url);
 
       Ok(CachedModule {
         is_remote,
         maybe_dependencies: None,
         maybe_emit,
         maybe_emit_path,
-        maybe_types: source_file.types_header,
+        maybe_types: source_file.maybe_types,
         maybe_version,
         media_type: source_file.media_type,
         requested_specifier,
-        source: source_file.source_code,
-        source_path: source_file.filename,
-        specifier,
+        source: source_file.source,
+        source_path: source_file.local,
+        specifier: source_file.specifier,
       })
     }
     .boxed_local()
@@ -521,6 +535,7 @@ impl SpecifierHandler for MemoryHandler {
 #[cfg(test)]
 pub mod tests {
   use super::*;
+  use crate::file_fetcher::CacheSetting;
   use crate::http_cache::HttpCache;
   use tempfile::TempDir;
 
@@ -541,12 +556,10 @@ pub mod tests {
     let deno_dir = DenoDir::new(Some(temp_dir.path().to_path_buf()))
       .expect("could not setup");
 
-    let file_fetcher = SourceFileFetcher::new(
+    let file_fetcher = FileFetcher::new(
       HttpCache::new(&temp_dir.path().to_path_buf().join("deps")),
+      CacheSetting::Use,
       true,
-      Vec::new(),
-      false,
-      false,
       None,
     )
     .expect("could not setup");

cli/tests/035_cached_only_flag.out

@@ -1 +1 @@
-error: Cannot find module "http://127.0.0.1:4545/cli/tests/019_media_types.ts" in cache, --cached-only is specified
+error: Specifier not found in cache: "http://127.0.0.1:4545/cli/tests/019_media_types.ts", --cached-only is specified.

cli/tests/052_no_remote_flag.out

@@ -1 +1 @@
-error: Cannot resolve module "http://127.0.0.1:4545/cli/tests/019_media_types.ts"
+error: A remote specifier was requested: "http://127.0.0.1:4545/cli/tests/019_media_types.ts", but --no-remote is specified.

cli/tests/error_004_missing_module.ts.out

@@ -1,2 +1,2 @@
-[WILDCARD]error: Cannot resolve module "file:///[WILDCARD]cli/tests/bad-module.ts" from "file:///[WILDCARD]cli/tests/error_004_missing_module.ts"
+[WILDCARD]error: Cannot resolve module "file:///[WILDCARD]cli/tests/bad-module.ts" from "file:///[WILDCARD]cli/tests/error_004_missing_module.ts".
     at file:///[WILDCARD]cli/tests/error_004_missing_module.ts:2:0

cli/tests/error_005_missing_dynamic_import.ts.out

@@ -1 +1,2 @@
-error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts"
+error: Cannot resolve module "[WILDCARD]/bad-module.ts" from "[WILDCARD]/error_005_missing_dynamic_import.ts".
+    at file:///[WILDCARD]/error_005_missing_dynamic_import.ts:3:26

cli/tests/error_006_import_ext_failure.ts.out

@@ -1,2 +1,2 @@
-[WILDCARD]error: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts"
+[WILDCARD]error: Cannot resolve module "[WILDCARD]/non-existent" from "[WILDCARD]/error_006_import_ext_failure.ts".
     at file:///[WILDCARD]cli/tests/error_006_import_ext_failure.ts:1:0

cli/tests/error_013_missing_script.out

@@ -1 +1 @@
-error: Cannot resolve module "[WILDCARD]missing_file_name"
+error: Cannot resolve module "[WILDCARD]missing_file_name".

cli/tests/error_015_dynamic_import_permissions.out

@@ -1 +1,2 @@
 error: network access to "http://localhost:4545/cli/tests/subdir/mod4.js", run again with the --allow-net flag
+    at file:///[WILDCARD]cli/tests/error_015_dynamic_import_permissions.js:[WILDCARD]

cli/tests/type_directives_js_main.js.out

@@ -1,3 +1,3 @@
 [WILDCARD]
-DEBUG RS - [WILDCARD] - fetch_source_file specifier: file:[WILDCARD]cli/tests/subdir/type_reference.d.ts [WILDCARD]
+DEBUG RS - [WILDCARD] - FileFetcher::fetch() - specifier: file:///[WILDCARD]cli/tests/subdir/type_reference.d.ts
 [WILDCARD]