2024-01-01 19:58:21 +00:00
|
|
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
2021-10-10 21:26:22 +00:00
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
use crate::args::CacheSetting;
|
2021-12-22 13:25:06 +00:00
|
|
|
use crate::errors::get_error_class_name;
|
2023-09-07 13:09:16 +00:00
|
|
|
use crate::file_fetcher::FetchOptions;
|
2021-10-10 21:26:22 +00:00
|
|
|
use crate::file_fetcher::FileFetcher;
|
2023-10-25 18:39:00 +00:00
|
|
|
use crate::npm::CliNpmResolver;
|
2023-08-08 14:23:02 +00:00
|
|
|
use crate::util::fs::atomic_write_file;
|
2021-10-10 21:26:22 +00:00
|
|
|
|
2023-09-07 13:09:16 +00:00
|
|
|
use deno_ast::MediaType;
|
2023-02-22 22:21:05 +00:00
|
|
|
use deno_core::futures;
|
2021-10-10 21:26:22 +00:00
|
|
|
use deno_core::futures::FutureExt;
|
|
|
|
use deno_core::ModuleSpecifier;
|
|
|
|
use deno_graph::source::CacheInfo;
|
|
|
|
use deno_graph::source::LoadFuture;
|
|
|
|
use deno_graph::source::LoadResponse;
|
|
|
|
use deno_graph::source::Loader;
|
2023-01-07 16:25:34 +00:00
|
|
|
use deno_runtime::permissions::PermissionsContainer;
|
2023-03-22 14:15:53 +00:00
|
|
|
use std::collections::HashMap;
|
2023-08-08 14:23:02 +00:00
|
|
|
use std::path::Path;
|
2023-08-02 00:49:09 +00:00
|
|
|
use std::path::PathBuf;
|
2021-10-10 21:26:22 +00:00
|
|
|
use std::sync::Arc;
|
2023-08-08 14:23:02 +00:00
|
|
|
use std::time::SystemTime;
|
2021-10-10 21:26:22 +00:00
|
|
|
|
feat(core): initialize SQLite off-main-thread (#18401)
This gets SQLite off the flamegraph and reduces initialization time by
somewhere between 0.2ms and 0.5ms. In addition, I took the opportunity
to move all the cache management code to a single place and reduce
duplication. While the PR has a net gain of lines, much of that is just
being a bit more deliberate with how we're recovering from errors.
The existing caches had various policies for dealing with cache
corruption, so I've unified them and tried to isolate the decisions we
make for recovery in a single place (see `open_connection` in
`CacheDB`). The policy I chose was:
1. Retry twice to open on-disk caches
2. If that fails, try to delete the file and recreate it on-disk
3. If we fail to delete the file or re-create a new cache, use a
fallback strategy that can be chosen per-cache: InMemory (temporary
cache for the process run), BlackHole (ignore writes, return empty
reads), or Error (fail on every operation).
The caches all use the same general code now, and share the cache
failure recovery policy.
In addition, it cleans up a TODO in the `NodeAnalysisCache`.
2023-03-27 22:01:52 +00:00
|
|
|
mod cache_db;
|
|
|
|
mod caches;
|
2022-07-12 22:58:39 +00:00
|
|
|
mod check;
|
2024-04-17 14:19:55 +00:00
|
|
|
mod code_cache;
|
2022-07-12 22:58:39 +00:00
|
|
|
mod common;
|
2022-11-26 00:04:30 +00:00
|
|
|
mod deno_dir;
|
2022-07-12 22:58:39 +00:00
|
|
|
mod disk_cache;
|
|
|
|
mod emit;
|
2024-02-20 21:29:57 +00:00
|
|
|
mod fast_check;
|
2022-07-12 22:58:39 +00:00
|
|
|
mod incremental;
|
2023-10-25 22:13:22 +00:00
|
|
|
mod module_info;
|
2022-10-01 10:15:56 +00:00
|
|
|
mod node;
|
2022-08-22 16:14:59 +00:00
|
|
|
mod parsed_source;
|
2022-07-12 22:58:39 +00:00
|
|
|
|
feat(core): initialize SQLite off-main-thread (#18401)
This gets SQLite off the flamegraph and reduces initialization time by
somewhere between 0.2ms and 0.5ms. In addition, I took the opportunity
to move all the cache management code to a single place and reduce
duplication. While the PR has a net gain of lines, much of that is just
being a bit more deliberate with how we're recovering from errors.
The existing caches had various policies for dealing with cache
corruption, so I've unified them and tried to isolate the decisions we
make for recovery in a single place (see `open_connection` in
`CacheDB`). The policy I chose was:
1. Retry twice to open on-disk caches
2. If that fails, try to delete the file and recreate it on-disk
3. If we fail to delete the file or re-create a new cache, use a
fallback strategy that can be chosen per-cache: InMemory (temporary
cache for the process run), BlackHole (ignore writes, return empty
reads), or Error (fail on every operation).
The caches all use the same general code now, and share the cache
failure recovery policy.
In addition, it cleans up a TODO in the `NodeAnalysisCache`.
2023-03-27 22:01:52 +00:00
|
|
|
pub use caches::Caches;
|
2022-07-12 22:58:39 +00:00
|
|
|
pub use check::TypeCheckCache;
|
2024-04-17 14:19:55 +00:00
|
|
|
pub use code_cache::CodeCache;
|
2022-07-19 15:58:18 +00:00
|
|
|
pub use common::FastInsecureHasher;
|
2022-11-26 00:04:30 +00:00
|
|
|
pub use deno_dir::DenoDir;
|
2023-05-25 18:27:45 +00:00
|
|
|
pub use deno_dir::DenoDirProvider;
|
2022-07-12 22:58:39 +00:00
|
|
|
pub use disk_cache::DiskCache;
|
|
|
|
pub use emit::EmitCache;
|
2024-02-20 21:29:57 +00:00
|
|
|
pub use fast_check::FastCheckCache;
|
2022-07-12 22:58:39 +00:00
|
|
|
pub use incremental::IncrementalCache;
|
2023-10-25 22:13:22 +00:00
|
|
|
pub use module_info::ModuleInfoCache;
|
2022-10-01 10:15:56 +00:00
|
|
|
pub use node::NodeAnalysisCache;
|
2024-02-06 20:57:10 +00:00
|
|
|
pub use parsed_source::LazyGraphSourceParser;
|
2022-08-22 16:14:59 +00:00
|
|
|
pub use parsed_source::ParsedSourceCache;
|
2022-07-12 22:58:39 +00:00
|
|
|
|
2022-11-28 22:28:54 +00:00
|
|
|
/// Permissions used to save a file in the disk caches.
|
|
|
|
pub const CACHE_PERM: u32 = 0o644;
|
|
|
|
|
2023-08-08 14:23:02 +00:00
|
|
|
#[derive(Debug, Clone)]
|
|
|
|
pub struct RealDenoCacheEnv;
|
|
|
|
|
|
|
|
impl deno_cache_dir::DenoCacheEnv for RealDenoCacheEnv {
|
|
|
|
fn read_file_bytes(&self, path: &Path) -> std::io::Result<Option<Vec<u8>>> {
|
|
|
|
match std::fs::read(path) {
|
|
|
|
Ok(s) => Ok(Some(s)),
|
|
|
|
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
|
|
|
Err(err) => Err(err),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn atomic_write_file(
|
|
|
|
&self,
|
|
|
|
path: &Path,
|
|
|
|
bytes: &[u8],
|
|
|
|
) -> std::io::Result<()> {
|
|
|
|
atomic_write_file(path, bytes, CACHE_PERM)
|
|
|
|
}
|
|
|
|
|
|
|
|
fn modified(&self, path: &Path) -> std::io::Result<Option<SystemTime>> {
|
|
|
|
match std::fs::metadata(path) {
|
|
|
|
Ok(metadata) => Ok(Some(
|
|
|
|
metadata.modified().unwrap_or_else(|_| SystemTime::now()),
|
|
|
|
)),
|
|
|
|
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(None),
|
|
|
|
Err(err) => Err(err),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn is_file(&self, path: &Path) -> bool {
|
|
|
|
path.is_file()
|
|
|
|
}
|
|
|
|
|
|
|
|
fn time_now(&self) -> SystemTime {
|
|
|
|
SystemTime::now()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub type GlobalHttpCache = deno_cache_dir::GlobalHttpCache<RealDenoCacheEnv>;
|
|
|
|
pub type LocalHttpCache = deno_cache_dir::LocalHttpCache<RealDenoCacheEnv>;
|
|
|
|
pub type LocalLspHttpCache =
|
|
|
|
deno_cache_dir::LocalLspHttpCache<RealDenoCacheEnv>;
|
|
|
|
pub use deno_cache_dir::HttpCache;
|
|
|
|
|
2024-01-14 01:56:56 +00:00
|
|
|
use self::module_info::ModuleInfoCacheSourceHash;
|
|
|
|
|
2021-10-10 21:26:22 +00:00
|
|
|
/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
|
|
|
|
/// a concise interface to the DENO_DIR when building module graphs.
|
2022-03-23 13:54:22 +00:00
|
|
|
pub struct FetchCacher {
|
2022-07-19 15:58:18 +00:00
|
|
|
emit_cache: EmitCache,
|
2021-10-10 21:26:22 +00:00
|
|
|
file_fetcher: Arc<FileFetcher>,
|
2023-03-22 14:15:53 +00:00
|
|
|
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
|
2023-08-02 00:49:09 +00:00
|
|
|
global_http_cache: Arc<GlobalHttpCache>,
|
2023-10-25 18:39:00 +00:00
|
|
|
npm_resolver: Arc<dyn CliNpmResolver>,
|
2023-10-25 22:13:22 +00:00
|
|
|
module_info_cache: Arc<ModuleInfoCache>,
|
2023-04-26 20:23:28 +00:00
|
|
|
permissions: PermissionsContainer,
|
2023-02-09 00:45:04 +00:00
|
|
|
cache_info_enabled: bool,
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl FetchCacher {
|
|
|
|
pub fn new(
|
2022-07-19 15:58:18 +00:00
|
|
|
emit_cache: EmitCache,
|
2023-02-03 19:15:16 +00:00
|
|
|
file_fetcher: Arc<FileFetcher>,
|
2023-03-22 14:15:53 +00:00
|
|
|
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
|
2023-08-02 00:49:09 +00:00
|
|
|
global_http_cache: Arc<GlobalHttpCache>,
|
2023-10-25 18:39:00 +00:00
|
|
|
npm_resolver: Arc<dyn CliNpmResolver>,
|
2023-10-25 22:13:22 +00:00
|
|
|
module_info_cache: Arc<ModuleInfoCache>,
|
2023-04-26 20:23:28 +00:00
|
|
|
permissions: PermissionsContainer,
|
2021-10-10 21:26:22 +00:00
|
|
|
) -> Self {
|
|
|
|
Self {
|
2022-07-19 15:58:18 +00:00
|
|
|
emit_cache,
|
2021-10-10 21:26:22 +00:00
|
|
|
file_fetcher,
|
2023-03-22 14:15:53 +00:00
|
|
|
file_header_overrides,
|
2023-08-02 00:49:09 +00:00
|
|
|
global_http_cache,
|
2023-10-25 18:39:00 +00:00
|
|
|
npm_resolver,
|
2023-10-25 22:13:22 +00:00
|
|
|
module_info_cache,
|
2023-04-26 20:23:28 +00:00
|
|
|
permissions,
|
2023-02-09 00:45:04 +00:00
|
|
|
cache_info_enabled: false,
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|
|
|
|
}
|
2023-02-09 00:45:04 +00:00
|
|
|
|
|
|
|
/// The cache information takes a bit of time to fetch and it's
|
|
|
|
/// not always necessary. It should only be enabled for deno info.
|
|
|
|
pub fn enable_loading_cache_info(&mut self) {
|
|
|
|
self.cache_info_enabled = true;
|
|
|
|
}
|
2023-08-02 00:49:09 +00:00
|
|
|
|
|
|
|
// DEPRECATED: Where the file is stored and how it's stored should be an implementation
|
|
|
|
// detail of the cache.
|
|
|
|
//
|
|
|
|
// todo(dsheret): remove once implementing
|
|
|
|
// * https://github.com/denoland/deno/issues/17707
|
|
|
|
// * https://github.com/denoland/deno/issues/17703
|
|
|
|
#[deprecated(
|
|
|
|
note = "There should not be a way to do this because the file may not be cached at a local path in the future."
|
|
|
|
)]
|
|
|
|
fn get_local_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> {
|
|
|
|
// TODO(@kitsonk) fix when deno_graph does not query cache for synthetic
|
|
|
|
// modules
|
|
|
|
if specifier.scheme() == "flags" {
|
|
|
|
None
|
|
|
|
} else if specifier.scheme() == "file" {
|
|
|
|
specifier.to_file_path().ok()
|
|
|
|
} else {
|
|
|
|
#[allow(deprecated)]
|
|
|
|
self
|
|
|
|
.global_http_cache
|
|
|
|
.get_global_cache_filepath(specifier)
|
|
|
|
.ok()
|
|
|
|
}
|
|
|
|
}
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl Loader for FetchCacher {
|
|
|
|
fn get_cache_info(&self, specifier: &ModuleSpecifier) -> Option<CacheInfo> {
|
2023-02-09 00:45:04 +00:00
|
|
|
if !self.cache_info_enabled {
|
|
|
|
return None;
|
|
|
|
}
|
|
|
|
|
2023-07-08 20:06:45 +00:00
|
|
|
#[allow(deprecated)]
|
2023-08-02 00:49:09 +00:00
|
|
|
let local = self.get_local_path(specifier)?;
|
2021-10-10 21:26:22 +00:00
|
|
|
if local.is_file() {
|
|
|
|
let emit = self
|
2022-07-19 15:58:18 +00:00
|
|
|
.emit_cache
|
|
|
|
.get_emit_filepath(specifier)
|
2021-10-10 21:26:22 +00:00
|
|
|
.filter(|p| p.is_file());
|
|
|
|
Some(CacheInfo {
|
|
|
|
local: Some(local),
|
|
|
|
emit,
|
2022-07-19 15:58:18 +00:00
|
|
|
map: None,
|
2021-10-10 21:26:22 +00:00
|
|
|
})
|
|
|
|
} else {
|
|
|
|
None
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn load(
|
|
|
|
&mut self,
|
|
|
|
specifier: &ModuleSpecifier,
|
2024-02-15 19:49:35 +00:00
|
|
|
options: deno_graph::source::LoadOptions,
|
2021-10-10 21:26:22 +00:00
|
|
|
) -> LoadFuture {
|
2023-09-07 13:09:16 +00:00
|
|
|
use deno_graph::source::CacheSetting as LoaderCacheSetting;
|
|
|
|
|
2024-03-06 00:23:51 +00:00
|
|
|
if specifier.scheme() == "file"
|
|
|
|
&& specifier.path().contains("/node_modules/")
|
|
|
|
{
|
2023-02-23 15:58:10 +00:00
|
|
|
// The specifier might be in a completely different symlinked tree than
|
2023-10-25 18:39:00 +00:00
|
|
|
// what the node_modules url is in (ex. `/my-project-1/node_modules`
|
|
|
|
// symlinked to `/my-project-2/node_modules`), so first we checked if the path
|
|
|
|
// is in a node_modules dir to avoid needlessly canonicalizing, then now compare
|
2023-02-23 15:58:10 +00:00
|
|
|
// against the canonicalized specifier.
|
2023-10-25 18:39:00 +00:00
|
|
|
let specifier =
|
|
|
|
crate::node::resolve_specifier_into_node_modules(specifier);
|
|
|
|
if self.npm_resolver.in_npm_package(&specifier) {
|
|
|
|
return Box::pin(futures::future::ready(Ok(Some(
|
|
|
|
LoadResponse::External { specifier },
|
|
|
|
))));
|
2023-02-22 22:21:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-10-10 21:26:22 +00:00
|
|
|
let file_fetcher = self.file_fetcher.clone();
|
2023-03-22 14:15:53 +00:00
|
|
|
let file_header_overrides = self.file_header_overrides.clone();
|
2023-09-07 13:09:16 +00:00
|
|
|
let permissions = self.permissions.clone();
|
2023-02-22 19:15:25 +00:00
|
|
|
let specifier = specifier.clone();
|
2021-10-10 21:26:22 +00:00
|
|
|
|
|
|
|
async move {
|
2024-02-15 19:49:35 +00:00
|
|
|
let maybe_cache_setting = match options.cache_setting {
|
2023-09-07 13:09:16 +00:00
|
|
|
LoaderCacheSetting::Use => None,
|
|
|
|
LoaderCacheSetting::Reload => {
|
|
|
|
if matches!(file_fetcher.cache_setting(), CacheSetting::Only) {
|
|
|
|
return Err(deno_core::anyhow::anyhow!(
|
|
|
|
"Failed to resolve version constraint. Try running again without --cached-only"
|
|
|
|
));
|
|
|
|
}
|
|
|
|
Some(CacheSetting::ReloadAll)
|
|
|
|
}
|
|
|
|
LoaderCacheSetting::Only => Some(CacheSetting::Only),
|
|
|
|
};
|
2022-01-13 16:58:00 +00:00
|
|
|
file_fetcher
|
2023-09-07 13:09:16 +00:00
|
|
|
.fetch_with_options(FetchOptions {
|
|
|
|
specifier: &specifier,
|
|
|
|
permissions,
|
|
|
|
maybe_accept: None,
|
|
|
|
maybe_cache_setting: maybe_cache_setting.as_ref(),
|
2024-02-15 19:49:35 +00:00
|
|
|
maybe_checksum: options.maybe_checksum,
|
2023-09-07 13:09:16 +00:00
|
|
|
})
|
2021-10-10 21:26:22 +00:00
|
|
|
.await
|
2023-03-15 21:46:36 +00:00
|
|
|
.map(|file| {
|
2023-03-22 14:15:53 +00:00
|
|
|
let maybe_headers =
|
|
|
|
match (file.maybe_headers, file_header_overrides.get(&specifier)) {
|
|
|
|
(Some(headers), Some(overrides)) => {
|
|
|
|
Some(headers.into_iter().chain(overrides.clone()).collect())
|
|
|
|
}
|
|
|
|
(Some(headers), None) => Some(headers),
|
|
|
|
(None, Some(overrides)) => Some(overrides.clone()),
|
|
|
|
(None, None) => None,
|
|
|
|
};
|
2023-03-15 21:46:36 +00:00
|
|
|
Ok(Some(LoadResponse::Module {
|
|
|
|
specifier: file.specifier,
|
2023-03-22 14:15:53 +00:00
|
|
|
maybe_headers,
|
2023-03-15 21:46:36 +00:00
|
|
|
content: file.source,
|
|
|
|
}))
|
|
|
|
})
|
|
|
|
.unwrap_or_else(|err| {
|
2023-09-18 14:46:44 +00:00
|
|
|
if let Some(io_err) = err.downcast_ref::<std::io::Error>() {
|
|
|
|
if io_err.kind() == std::io::ErrorKind::NotFound {
|
2021-12-22 13:25:06 +00:00
|
|
|
return Ok(None);
|
2023-09-18 14:46:44 +00:00
|
|
|
} else {
|
|
|
|
return Err(err);
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|
2023-03-15 21:46:36 +00:00
|
|
|
}
|
2023-09-18 14:46:44 +00:00
|
|
|
let error_class_name = get_error_class_name(&err);
|
|
|
|
match error_class_name {
|
|
|
|
"NotFound" => Ok(None),
|
2024-02-15 19:49:35 +00:00
|
|
|
"NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None),
|
2023-09-18 14:46:44 +00:00
|
|
|
_ => Err(err),
|
|
|
|
}
|
2023-03-15 21:46:36 +00:00
|
|
|
})
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|
|
|
|
.boxed()
|
|
|
|
}
|
2023-09-07 13:09:16 +00:00
|
|
|
|
|
|
|
fn cache_module_info(
|
|
|
|
&mut self,
|
|
|
|
specifier: &ModuleSpecifier,
|
2024-02-01 03:15:22 +00:00
|
|
|
source: &Arc<[u8]>,
|
2023-09-07 13:09:16 +00:00
|
|
|
module_info: &deno_graph::ModuleInfo,
|
|
|
|
) {
|
2024-02-01 03:15:22 +00:00
|
|
|
let source_hash = ModuleInfoCacheSourceHash::from_source(source);
|
2023-10-25 22:13:22 +00:00
|
|
|
let result = self.module_info_cache.set_module_info(
|
2023-09-07 13:09:16 +00:00
|
|
|
specifier,
|
|
|
|
MediaType::from_specifier(specifier),
|
2024-01-14 01:56:56 +00:00
|
|
|
&source_hash,
|
2023-09-07 13:09:16 +00:00
|
|
|
module_info,
|
|
|
|
);
|
|
|
|
if let Err(err) = result {
|
|
|
|
log::debug!(
|
|
|
|
"Error saving module cache info for {}. {:#}",
|
|
|
|
specifier,
|
|
|
|
err
|
|
|
|
);
|
|
|
|
}
|
|
|
|
}
|
2021-10-10 21:26:22 +00:00
|
|
|
}
|