development/deno
development/deno
1
0
Fork 0
mirror of https://github.com/denoland/deno synced 2024-08-27 03:50:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(npm): create general use NpmPackageResolver (#15882)

Browse source
This commit is contained in:
David Sherret 2022-09-13 11:59:01 -04:00 committed by GitHub
parent 51ba4764d1
commit 73efed218f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 445 additions and 472 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cli/main.rs
View file

@ -133,7 +133,7 @@ fn print_cache_info(
) -> Result<(), AnyError> {
let deno_dir = &state.dir.root;
let modules_cache = &state.file_fetcher.get_http_cache_location();
let npm_cache = &state.npm_resolver.get_cache_location();
let npm_cache = &state.npm_cache.as_readonly().get_cache_location();
let typescript_cache = &state.dir.gen_cache.location;
let registry_cache =
&state.dir.root.join(lsp::language_server::REGISTRIES_PATH);

1
cli/module_loader.rs
View file

@ -4,7 +4,6 @@ use crate::emit::emit_parsed_source;
use crate::emit::TsTypeLib;
use crate::graph_util::ModuleEntry;
use crate::node;
use crate::npm::NpmPackageResolver;
use crate::proc_state::ProcState;
use crate::text_encoding::code_without_source_map;
use crate::text_encoding::source_map_from_code;

19
cli/node/mod.rs
View file

@ -24,9 +24,9 @@ use deno_runtime::deno_node::legacy_main_resolve;
use deno_runtime::deno_node::package_exports_resolve;
use deno_runtime::deno_node::package_imports_resolve;
use deno_runtime::deno_node::package_resolve;
use deno_runtime::deno_node::DenoDirNpmResolver;
use deno_runtime::deno_node::NodeModuleKind;
use deno_runtime::deno_node::PackageJson;
use deno_runtime::deno_node::RequireNpmResolver;
use deno_runtime::deno_node::DEFAULT_CONDITIONS;
use deno_runtime::deno_node::NODE_GLOBAL_THIS_NAME;
use once_cell::sync::Lazy;
@ -34,7 +34,6 @@ use path_clean::PathClean;
use regex::Regex;
use crate::file_fetcher::FileFetcher;
use crate::npm::GlobalNpmPackageResolver;
use crate::npm::NpmPackageReference;
use crate::npm::NpmPackageReq;
use crate::npm::NpmPackageResolver;
@ -380,7 +379,7 @@ pub async fn initialize_binary_command(
pub fn node_resolve(
specifier: &str,
referrer: &ModuleSpecifier,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<Option<NodeResolution>, AnyError> {
// Note: if we are here, then the referrer is an esm module
// TODO(bartlomieju): skipped "policy" part as we don't plan to support it
@ -431,7 +430,7 @@ pub fn node_resolve(
pub fn node_resolve_npm_reference(
reference: &NpmPackageReference,
npm_resolver: &GlobalNpmPackageResolver,
npm_resolver: &NpmPackageResolver,
) -> Result<Option<NodeResolution>, AnyError> {
let package_folder = npm_resolver
.resolve_package_from_deno_module(&reference.req)?
@ -460,7 +459,7 @@ pub fn node_resolve_npm_reference(
pub fn node_resolve_binary_export(
pkg_req: &NpmPackageReq,
bin_name: Option<&str>,
npm_resolver: &GlobalNpmPackageResolver,
npm_resolver: &NpmPackageResolver,
) -> Result<NodeResolution, AnyError> {
let pkg = npm_resolver.resolve_package_from_deno_module(pkg_req)?;
let package_folder = pkg.folder_path;
@ -541,7 +540,7 @@ pub fn load_cjs_module_from_ext_node(
fn package_config_resolve(
package_subpath: &str,
package_dir: &Path,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
referrer_kind: NodeModuleKind,
) -> Result<PathBuf, AnyError> {
let package_json_path = package_dir.join("package.json");
@ -568,7 +567,7 @@ fn package_config_resolve(
fn url_to_node_resolution(
url: ModuleSpecifier,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<NodeResolution, AnyError> {
Ok(if url.as_str().starts_with("http") {
NodeResolution::Esm(url)
@ -640,7 +639,7 @@ fn module_resolve(
specifier: &str,
referrer: &ModuleSpecifier,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<Option<ModuleSpecifier>, AnyError> {
// note: if we're here, the referrer is an esm module
let url = if should_be_treated_as_relative_or_absolute_path(specifier) {
@ -721,7 +720,7 @@ pub fn translate_cjs_to_esm(
specifier: &ModuleSpecifier,
code: String,
media_type: MediaType,
npm_resolver: &GlobalNpmPackageResolver,
npm_resolver: &NpmPackageResolver,
) -> Result<String, AnyError> {
fn perform_cjs_analysis(
specifier: &str,
@ -843,7 +842,7 @@ fn resolve(
specifier: &str,
referrer: &ModuleSpecifier,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
if specifier.starts_with('/') {
todo!();

2
cli/npm/cache.rs
View file

@ -18,10 +18,10 @@ use crate::file_fetcher::CacheSetting;
use crate::fs_util;
use crate::progress_bar::ProgressBar;
use super::registry::NpmPackageVersionDistInfo;
use super::semver::NpmVersion;
use super::tarball::verify_and_extract_tarball;
use super::NpmPackageId;
use super::NpmPackageVersionDistInfo;
pub const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock";

416
cli/npm/mod.rs
View file

@ -3,422 +3,14 @@
mod cache;
mod registry;
mod resolution;
mod resolvers;
mod semver;
mod tarball;
use std::io::ErrorKind;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::bail;
use deno_core::anyhow::Context;
use deno_core::error::custom_error;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::url::Url;
use deno_runtime::deno_node::DenoDirNpmResolver;
pub use cache::NpmCache;
pub use registry::NpmRegistryApi;
pub use resolution::NpmPackageId;
pub use resolution::NpmPackageReference;
pub use resolution::NpmPackageReq;
pub use resolution::NpmResolutionPackage;
use cache::NpmCache;
use registry::NpmPackageVersionDistInfo;
use registry::NpmRegistryApi;
use resolution::NpmResolution;
use crate::deno_dir::DenoDir;
use crate::file_fetcher::CacheSetting;
use crate::progress_bar::ProgressBar;
use self::cache::ReadonlyNpmCache;
use self::resolution::NpmResolutionSnapshot;
/// Information about the local npm package.
pub struct LocalNpmPackageInfo {
/// Unique identifier.
pub id: NpmPackageId,
/// Local folder path of the npm package.
pub folder_path: PathBuf,
}
pub trait NpmPackageResolver {
/// Resolves an npm package from a Deno module.
fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError>;
/// Resolves an npm package from an npm package referrer.
fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
/// Resolve the root folder of the package the provided specifier is in.
///
/// This will error when the provided specifier is not in an npm package.
fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
/// Gets if the provided specifier is in an npm package.
fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
self.resolve_package_from_specifier(specifier).is_ok()
}
}
#[derive(Debug, Clone)]
pub struct GlobalNpmPackageResolver {
cache: NpmCache,
resolution: Arc<NpmResolution>,
registry_url: Url,
unstable: bool,
no_npm: bool,
}
impl GlobalNpmPackageResolver {
pub fn from_deno_dir(
dir: &DenoDir,
reload: bool,
cache_setting: CacheSetting,
unstable: bool,
no_npm: bool,
progress_bar: ProgressBar,
) -> Self {
Self::from_cache(
NpmCache::from_deno_dir(dir, cache_setting.clone(), progress_bar.clone()),
reload,
cache_setting,
unstable,
no_npm,
progress_bar,
)
}
fn from_cache(
cache: NpmCache,
reload: bool,
cache_setting: CacheSetting,
unstable: bool,
no_npm: bool,
progress_bar: ProgressBar,
) -> Self {
let api =
NpmRegistryApi::new(cache.clone(), reload, cache_setting, progress_bar);
let registry_url = api.base_url().to_owned();
let resolution = Arc::new(NpmResolution::new(api));
Self {
cache,
resolution,
registry_url,
unstable,
no_npm,
}
}
/// If the resolver has resolved any npm packages.
pub fn has_packages(&self) -> bool {
self.resolution.has_packages()
}
/// Adds a package requirement to the resolver.
pub async fn add_package_reqs(
&self,
packages: Vec<NpmPackageReq>,
) -> Result<(), AnyError> {
assert!(!packages.is_empty());
if !self.unstable {
bail!(
"Unstable use of npm specifiers. The --unstable flag must be provided."
)
}
if self.no_npm {
let fmt_reqs = packages
.iter()
.map(|p| format!("\"{}\"", p))
.collect::<Vec<_>>()
.join(", ");
return Err(custom_error(
"NoNpm",
format!(
"Following npm specifiers were requested: {}; but --no-npm is specified.",
fmt_reqs
),
));
}
self.resolution.add_package_reqs(packages).await
}
/// Caches all the packages in parallel.
pub async fn cache_packages(&self) -> Result<(), AnyError> {
if std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") == Ok("1".to_string()) {
// for some of the tests, we want downloading of packages
// to be deterministic so that the output is always the same
let mut packages = self.resolution.all_packages();
packages.sort_by(|a, b| a.id.cmp(&b.id));
for package in packages {
self
.cache
.ensure_package(&package.id, &package.dist, &self.registry_url)
.await
.with_context(|| {
format!("Failed caching npm package '{}'.", package.id)
})?;
}
} else {
let handles = self.resolution.all_packages().into_iter().map(|package| {
let cache = self.cache.clone();
let registry_url = self.registry_url.clone();
tokio::task::spawn(async move {
cache
.ensure_package(&package.id, &package.dist, &registry_url)
.await
.with_context(|| {
format!("Failed caching npm package '{}'.", package.id)
})
})
});
let results = futures::future::join_all(handles).await;
for result in results {
// surface the first error
result??;
}
}
Ok(())
}
fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo {
LocalNpmPackageInfo {
folder_path: self.cache.package_folder(id, &self.registry_url),
id: id.clone(),
}
}
/// Creates an inner clone.
#[allow(unused)]
pub fn snapshot(&self) -> NpmPackageResolverSnapshot {
NpmPackageResolverSnapshot {
cache: self.cache.as_readonly(),
snapshot: self.resolution.snapshot(),
registry_url: self.registry_url.clone(),
}
}
pub fn get_cache_location(&self) -> PathBuf {
self.cache.as_readonly().get_cache_location()
}
}
impl NpmPackageResolver for GlobalNpmPackageResolver {
fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let referrer_pkg_id = self
.cache
.resolve_package_id_from_specifier(referrer, &self.registry_url)?;
let pkg = self
.resolution
.resolve_package_from_package(name, &referrer_pkg_id)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg_id = self
.cache
.resolve_package_id_from_specifier(specifier, &self.registry_url)?;
Ok(self.local_package_info(&pkg_id))
}
}
#[derive(Clone, Debug)]
pub struct NpmPackageResolverSnapshot {
cache: ReadonlyNpmCache,
snapshot: NpmResolutionSnapshot,
registry_url: Url,
}
// todo(dsherret): implementing Default for this is error prone, but
// necessary for the LSP. We should remove this Default implementation.
// See comment on `ReadonlyNpmCache` for more details.
impl Default for NpmPackageResolverSnapshot {
fn default() -> Self {
Self {
cache: Default::default(),
snapshot: Default::default(),
registry_url: NpmRegistryApi::default_url(),
}
}
}
impl NpmPackageResolverSnapshot {
fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo {
LocalNpmPackageInfo {
folder_path: self.cache.package_folder(id, &self.registry_url),
id: id.clone(),
}
}
}
impl NpmPackageResolver for NpmPackageResolverSnapshot {
fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg = self.snapshot.resolve_package_from_deno_module(pkg_req)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let referrer_pkg_id = self
.cache
.resolve_package_id_from_specifier(referrer, &self.registry_url)?;
let pkg = self
.snapshot
.resolve_package_from_package(name, &referrer_pkg_id)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg_id = self
.cache
.resolve_package_id_from_specifier(specifier, &self.registry_url)?;
Ok(self.local_package_info(&pkg_id))
}
}
impl DenoDirNpmResolver for GlobalNpmPackageResolver {
fn resolve_package_folder_from_package(
&self,
specifier: &str,
referrer: &std::path::Path,
) -> Result<PathBuf, AnyError> {
let referrer = specifier_to_path(referrer)?;
self
.resolve_package_from_package(specifier, &referrer)
.map(|p| p.folder_path)
}
fn resolve_package_folder_from_path(
&self,
path: &Path,
) -> Result<PathBuf, AnyError> {
let specifier = specifier_to_path(path)?;
self
.resolve_package_from_specifier(&specifier)
.map(|p| p.folder_path)
}
fn in_npm_package(&self, path: &Path) -> bool {
let specifier = match ModuleSpecifier::from_file_path(path) {
Ok(p) => p,
Err(_) => return false,
};
self.resolve_package_from_specifier(&specifier).is_ok()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_read_permission(&registry_path, path)
}
}
impl DenoDirNpmResolver for NpmPackageResolverSnapshot {
fn resolve_package_folder_from_package(
&self,
specifier: &str,
referrer: &std::path::Path,
) -> Result<PathBuf, AnyError> {
let referrer = specifier_to_path(referrer)?;
self
.resolve_package_from_package(specifier, &referrer)
.map(|p| p.folder_path)
}
fn resolve_package_folder_from_path(
&self,
path: &Path,
) -> Result<PathBuf, AnyError> {
let specifier = specifier_to_path(path)?;
self
.resolve_package_from_specifier(&specifier)
.map(|p| p.folder_path)
}
fn in_npm_package(&self, path: &Path) -> bool {
let specifier = match ModuleSpecifier::from_file_path(path) {
Ok(p) => p,
Err(_) => return false,
};
self.resolve_package_from_specifier(&specifier).is_ok()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_read_permission(&registry_path, path)
}
}
fn specifier_to_path(path: &Path) -> Result<ModuleSpecifier, AnyError> {
match ModuleSpecifier::from_file_path(&path) {
Ok(specifier) => Ok(specifier),
Err(()) => bail!("Could not convert '{}' to url.", path.display()),
}
}
fn ensure_read_permission(
registry_path: &Path,
path: &Path,
) -> Result<(), AnyError> {
// allow reading if it's in the deno_dir node modules
if path.starts_with(&registry_path)
&& path
.components()
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}
Err(e) if e.kind() == ErrorKind::NotFound => {
return Ok(());
}
_ => {} // ignore
}
}
}
Err(deno_core::error::custom_error(
"PermissionDenied",
format!("Reading {} is not allowed", path.display()),
))
}
pub use resolvers::NpmPackageResolver;

15
cli/npm/registry.rs
View file

@ -131,21 +131,6 @@ impl NpmRegistryApi {
}
pub fn new(
cache: NpmCache,
reload: bool,
cache_setting: CacheSetting,
progress_bar: ProgressBar,
) -> Self {
Self::from_base(
Self::default_url(),
cache,
reload,
cache_setting,
progress_bar,
)
}
pub fn from_base(
base_url: Url,
cache: NpmCache,
reload: bool,

2
cli/npm/resolution.rs
View file

@ -471,6 +471,8 @@ impl NpmResolution {
!self.snapshot.read().packages.is_empty()
}
// todo(dsherret): for use in the lsp
#[allow(dead_code)]
pub fn snapshot(&self) -> NpmResolutionSnapshot {
self.snapshot.read().clone()
}

89
cli/npm/resolvers/common.rs Normal file
View file

@ -0,0 +1,89 @@
use std::path::Path;
use std::path::PathBuf;
use deno_ast::ModuleSpecifier;
use deno_core::anyhow::Context;
use deno_core::error::AnyError;
use deno_core::futures;
use deno_core::futures::future::BoxFuture;
use deno_core::url::Url;
use crate::npm::NpmCache;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmResolutionPackage;
/// Information about the local npm package.
pub struct LocalNpmPackageInfo {
/// Unique identifier.
pub id: NpmPackageId,
/// Local folder path of the npm package.
pub folder_path: PathBuf,
}
pub trait InnerNpmPackageResolver: Send + Sync {
fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError>;
fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError>;
fn has_packages(&self) -> bool;
fn add_package_reqs(
&self,
packages: Vec<NpmPackageReq>,
) -> BoxFuture<'static, Result<(), AnyError>>;
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError>;
}
/// Caches all the packages in parallel.
pub async fn cache_packages(
mut packages: Vec<NpmResolutionPackage>,
cache: &NpmCache,
registry_url: &Url,
) -> Result<(), AnyError> {
if std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") == Ok("1".to_string()) {
// for some of the tests, we want downloading of packages
// to be deterministic so that the output is always the same
packages.sort_by(|a, b| a.id.cmp(&b.id));
for package in packages {
cache
.ensure_package(&package.id, &package.dist, registry_url)
.await
.with_context(|| {
format!("Failed caching npm package '{}'.", package.id)
})?;
}
} else {
let handles = packages.into_iter().map(|package| {
let cache = cache.clone();
let registry_url = registry_url.clone();
tokio::task::spawn(async move {
cache
.ensure_package(&package.id, &package.dist, &registry_url)
.await
.with_context(|| {
format!("Failed caching npm package '{}'.", package.id)
})
})
});
let results = futures::future::join_all(handles).await;
for result in results {
// surface the first error
result??;
}
}
Ok(())
}

138
cli/npm/resolvers/global.rs Normal file
View file

@ -0,0 +1,138 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
use std::io::ErrorKind;
use std::path::Path;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_core::futures::future::BoxFuture;
use deno_core::futures::FutureExt;
use deno_core::url::Url;
use crate::npm::resolution::NpmResolution;
use crate::npm::resolvers::common::cache_packages;
use crate::npm::NpmCache;
use crate::npm::NpmPackageId;
use crate::npm::NpmPackageReq;
use crate::npm::NpmRegistryApi;
use super::common::InnerNpmPackageResolver;
use super::common::LocalNpmPackageInfo;
#[derive(Debug, Clone)]
pub struct GlobalNpmPackageResolver {
cache: NpmCache,
resolution: Arc<NpmResolution>,
registry_url: Url,
}
impl GlobalNpmPackageResolver {
pub fn new(cache: NpmCache, api: NpmRegistryApi) -> Self {
let registry_url = api.base_url().to_owned();
let resolution = Arc::new(NpmResolution::new(api));
Self {
cache,
resolution,
registry_url,
}
}
fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo {
LocalNpmPackageInfo {
folder_path: self.cache.package_folder(id, &self.registry_url),
id: id.clone(),
}
}
}
impl InnerNpmPackageResolver for GlobalNpmPackageResolver {
fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let referrer_pkg_id = self
.cache
.resolve_package_id_from_specifier(referrer, &self.registry_url)?;
let pkg = self
.resolution
.resolve_package_from_package(name, &referrer_pkg_id)?;
Ok(self.local_package_info(&pkg.id))
}
fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
let pkg_id = self
.cache
.resolve_package_id_from_specifier(specifier, &self.registry_url)?;
Ok(self.local_package_info(&pkg_id))
}
fn has_packages(&self) -> bool {
self.resolution.has_packages()
}
fn add_package_reqs(
&self,
packages: Vec<NpmPackageReq>,
) -> BoxFuture<'static, Result<(), AnyError>> {
let resolver = self.clone();
async move {
resolver.resolution.add_package_reqs(packages).await?;
cache_packages(
resolver.resolution.all_packages(),
&resolver.cache,
&resolver.registry_url,
)
.await
}
.boxed()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
let registry_path = self.cache.registry_folder(&self.registry_url);
ensure_read_permission(&registry_path, path)
}
}
fn ensure_read_permission(
registry_path: &Path,
path: &Path,
) -> Result<(), AnyError> {
// allow reading if it's in the deno_dir node modules
if path.starts_with(&registry_path)
&& path
.components()
.all(|c| !matches!(c, std::path::Component::ParentDir))
{
// todo(dsherret): cache this?
if let Ok(registry_path) = std::fs::canonicalize(registry_path) {
match std::fs::canonicalize(path) {
Ok(path) if path.starts_with(registry_path) => {
return Ok(());
}
Err(e) if e.kind() == ErrorKind::NotFound => {
return Ok(());
}
_ => {} // ignore
}
}
}
Err(deno_core::error::custom_error(
"PermissionDenied",
format!("Reading {} is not allowed", path.display()),
))
}

158
cli/npm/resolvers/mod.rs Normal file
View file

@ -0,0 +1,158 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
mod common;
mod global;
use deno_core::anyhow::bail;
use deno_core::error::custom_error;
use deno_runtime::deno_node::RequireNpmResolver;
use global::GlobalNpmPackageResolver;
use std::path::Path;
use std::path::PathBuf;
use std::sync::Arc;
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use self::common::InnerNpmPackageResolver;
use super::NpmCache;
use super::NpmPackageReq;
use super::NpmRegistryApi;
pub use self::common::LocalNpmPackageInfo;
#[derive(Clone)]
pub struct NpmPackageResolver {
unstable: bool,
no_npm: bool,
inner: Arc<dyn InnerNpmPackageResolver>,
}
impl NpmPackageResolver {
pub fn new(
cache: NpmCache,
api: NpmRegistryApi,
unstable: bool,
no_npm: bool,
) -> Self {
// For now, always create a GlobalNpmPackageResolver, but in the future
// this might be a local node_modules folder
let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api));
Self {
unstable,
no_npm,
inner,
}
}
/// Resolves an npm package from a Deno module.
pub fn resolve_package_from_deno_module(
&self,
pkg_req: &NpmPackageReq,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_deno_module(pkg_req)
}
/// Resolves an npm package from an npm package referrer.
pub fn resolve_package_from_package(
&self,
name: &str,
referrer: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_package(name, referrer)
}
/// Resolve the root folder of the package the provided specifier is in.
///
/// This will error when the provided specifier is not in an npm package.
pub fn resolve_package_from_specifier(
&self,
specifier: &ModuleSpecifier,
) -> Result<LocalNpmPackageInfo, AnyError> {
self.inner.resolve_package_from_specifier(specifier)
}
/// Gets if the provided specifier is in an npm package.
pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
self.resolve_package_from_specifier(specifier).is_ok()
}
/// If the resolver has resolved any npm packages.
pub fn has_packages(&self) -> bool {
self.inner.has_packages()
}
/// Adds a package requirement to the resolver and ensures everything is setup.
pub async fn add_package_reqs(
&self,
packages: Vec<NpmPackageReq>,
) -> Result<(), AnyError> {
assert!(!packages.is_empty());
if !self.unstable {
bail!(
"Unstable use of npm specifiers. The --unstable flag must be provided."
)
}
if self.no_npm {
let fmt_reqs = packages
.iter()
.map(|p| format!("\"{}\"", p))
.collect::<Vec<_>>()
.join(", ");
return Err(custom_error(
"NoNpm",
format!(
"Following npm specifiers were requested: {}; but --no-npm is specified.",
fmt_reqs
),
));
}
self.inner.add_package_reqs(packages).await
}
}
impl RequireNpmResolver for NpmPackageResolver {
fn resolve_package_folder_from_package(
&self,
specifier: &str,
referrer: &std::path::Path,
) -> Result<PathBuf, AnyError> {
let referrer = specifier_to_path(referrer)?;
self
.resolve_package_from_package(specifier, &referrer)
.map(|p| p.folder_path)
}
fn resolve_package_folder_from_path(
&self,
path: &Path,
) -> Result<PathBuf, AnyError> {
let specifier = specifier_to_path(path)?;
self
.resolve_package_from_specifier(&specifier)
.map(|p| p.folder_path)
}
fn in_npm_package(&self, path: &Path) -> bool {
let specifier = match ModuleSpecifier::from_file_path(path) {
Ok(p) => p,
Err(_) => return false,
};
self.resolve_package_from_specifier(&specifier).is_ok()
}
fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> {
self.inner.ensure_read_permission(path)
}
}
fn specifier_to_path(path: &Path) -> Result<ModuleSpecifier, AnyError> {
match ModuleSpecifier::from_file_path(&path) {
Ok(specifier) => Ok(specifier),
Err(()) => bail!("Could not convert '{}' to url.", path.display()),
}
}

24
cli/proc_state.rs
View file

@ -22,9 +22,10 @@ use crate::lockfile::as_maybe_locker;
use crate::lockfile::Lockfile;
use crate::node;
use crate::node::NodeResolution;
use crate::npm::GlobalNpmPackageResolver;
use crate::npm::NpmCache;
use crate::npm::NpmPackageReference;
use crate::npm::NpmPackageResolver;
use crate::npm::NpmRegistryApi;
use crate::progress_bar::ProgressBar;
use crate::resolver::ImportMapResolver;
use crate::resolver::JsxResolver;
@ -87,7 +88,8 @@ pub struct Inner {
pub parsed_source_cache: ParsedSourceCache,
maybe_resolver: Option<Arc<dyn deno_graph::source::Resolver + Send + Sync>>,
maybe_file_watcher_reporter: Option<FileWatcherReporter>,
pub npm_resolver: GlobalNpmPackageResolver,
pub npm_cache: NpmCache,
pub npm_resolver: NpmPackageResolver,
pub cjs_resolutions: Mutex<HashSet<ModuleSpecifier>>,
progress_bar: ProgressBar,
}
@ -220,15 +222,26 @@ impl ProcState {
let emit_cache = EmitCache::new(dir.gen_cache.clone());
let parsed_source_cache =
ParsedSourceCache::new(Some(dir.dep_analysis_db_file_path()));
let npm_resolver = GlobalNpmPackageResolver::from_deno_dir(
let registry_url = NpmRegistryApi::default_url();
let npm_cache = NpmCache::from_deno_dir(
&dir,
cli_options.cache_setting(),
progress_bar.clone(),
);
let api = NpmRegistryApi::new(
registry_url,
npm_cache.clone(),
cli_options.reload_flag(),
cli_options.cache_setting(),
progress_bar.clone(),
);
let npm_resolver = NpmPackageResolver::new(
npm_cache.clone(),
api,
cli_options.unstable()
// don't do the unstable error when in the lsp
|| matches!(cli_options.sub_command(), DenoSubcommand::Lsp),
cli_options.no_npm(),
progress_bar.clone(),
);
let emit_options: deno_ast::EmitOptions = ts_config_result.ts_config.into();
@ -253,6 +266,7 @@ impl ProcState {
parsed_source_cache,
maybe_resolver,
maybe_file_watcher_reporter,
npm_cache,
npm_resolver,
cjs_resolutions: Default::default(),
progress_bar,
@ -413,7 +427,6 @@ impl ProcState {
.npm_resolver
.add_package_reqs(npm_package_references)
.await?;
self.npm_resolver.cache_packages().await?;
self.prepare_node_std_graph().await?;
}
@ -650,7 +663,6 @@ impl ProcState {
}
if !package_reqs.is_empty() {
self.npm_resolver.add_package_reqs(package_reqs).await?;
self.npm_resolver.cache_packages().await?;
}
Ok(graph)

1
cli/worker.rs
View file

@ -356,7 +356,6 @@ pub async fn create_main_worker(
ps.npm_resolver
.add_package_reqs(vec![package_ref.req.clone()])
.await?;
ps.npm_resolver.cache_packages().await?;
ps.prepare_node_std_graph().await?;
let node_resolution = node::node_resolve_binary_export(
&package_ref.req,

20
ext/node/lib.rs
View file

@ -30,7 +30,7 @@ pub trait NodePermissions {
fn check_read(&mut self, path: &Path) -> Result<(), AnyError>;
}
pub trait DenoDirNpmResolver {
pub trait RequireNpmResolver {
fn resolve_package_folder_from_package(
&self,
specifier: &str,
@ -63,7 +63,7 @@ struct Unstable(pub bool);
pub fn init<P: NodePermissions + 'static>(
unstable: bool,
maybe_npm_resolver: Option<Rc<dyn DenoDirNpmResolver>>,
maybe_npm_resolver: Option<Rc<dyn RequireNpmResolver>>,
) -> Extension {
Extension::builder()
.js(include_js_files!(
@ -121,7 +121,7 @@ where
P: NodePermissions + 'static,
{
let resolver = {
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>();
resolver.clone()
};
if resolver.ensure_read_permission(file_path).is_ok() {
@ -287,7 +287,7 @@ fn op_require_resolve_deno_dir(
parent_filename: String,
) -> Option<String> {
check_unstable(state);
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>();
resolver
.resolve_package_folder_from_package(
&request,
@ -300,7 +300,7 @@ fn op_require_resolve_deno_dir(
#[op]
fn op_require_is_deno_dir_package(state: &mut OpState, path: String) -> bool {
check_unstable(state);
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>();
resolver.in_npm_package(&PathBuf::from(path))
}
@ -470,7 +470,7 @@ fn op_require_try_self(
return Ok(None);
}
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>().clone();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>().clone();
let pkg = resolution::get_package_scope_config(
&Url::from_file_path(parent_path.unwrap()).unwrap(),
&*resolver,
@ -552,7 +552,7 @@ fn op_require_resolve_exports(
parent_path: String,
) -> Result<Option<String>, AnyError> {
check_unstable(state);
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>().clone();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>().clone();
let pkg_path = if resolver.in_npm_package(&PathBuf::from(&modules_path)) {
modules_path
@ -594,7 +594,7 @@ where
state,
PathBuf::from(&filename).parent().unwrap(),
)?;
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>().clone();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>().clone();
resolution::get_closest_package_json(
&Url::from_file_path(filename).unwrap(),
&*resolver,
@ -607,7 +607,7 @@ fn op_require_read_package_scope(
package_json_path: String,
) -> Option<PackageJson> {
check_unstable(state);
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>().clone();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>().clone();
let package_json_path = PathBuf::from(package_json_path);
PackageJson::load(&*resolver, package_json_path).ok()
}
@ -624,7 +624,7 @@ where
check_unstable(state);
let parent_path = PathBuf::from(&parent_filename);
ensure_read_permission::<P>(state, &parent_path)?;
let resolver = state.borrow::<Rc<dyn DenoDirNpmResolver>>().clone();
let resolver = state.borrow::<Rc<dyn RequireNpmResolver>>().clone();
let pkg = PackageJson::load(&*resolver, parent_path.join("package.json"))?;
if pkg.imports.is_some() {

4
ext/node/package_json.rs
View file

@ -2,7 +2,7 @@
use crate::NodeModuleKind;
use super::DenoDirNpmResolver;
use super::RequireNpmResolver;
use deno_core::anyhow;
use deno_core::anyhow::bail;
use deno_core::error::AnyError;
@ -44,7 +44,7 @@ impl PackageJson {
}
pub fn load(
resolver: &dyn DenoDirNpmResolver,
resolver: &dyn RequireNpmResolver,
path: PathBuf,
) -> Result<PackageJson, AnyError> {
resolver.ensure_read_permission(&path)?;

18
ext/node/resolution.rs
View file

@ -15,7 +15,7 @@ use regex::Regex;
use crate::errors;
use crate::package_json::PackageJson;
use crate::DenoDirNpmResolver;
use crate::RequireNpmResolver;
pub static DEFAULT_CONDITIONS: &[&str] = &["deno", "node", "import"];
pub static REQUIRE_CONDITIONS: &[&str] = &["require", "node"];
@ -93,7 +93,7 @@ pub fn package_imports_resolve(
referrer: &ModuleSpecifier,
referrer_kind: NodeModuleKind,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
if name == "#" || name.starts_with("#/") || name.ends_with('/') {
let reason = "is not a valid internal imports specifier name";
@ -224,7 +224,7 @@ fn resolve_package_target_string(
pattern: bool,
internal: bool,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
if !subpath.is_empty() && !pattern && !target.ends_with('/') {
return Err(throw_invalid_package_target(
@ -326,7 +326,7 @@ fn resolve_package_target(
pattern: bool,
internal: bool,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<Option<PathBuf>, AnyError> {
if let Some(target) = target.as_str() {
return Ok(Some(resolve_package_target_string(
@ -441,7 +441,7 @@ pub fn package_exports_resolve(
referrer: &ModuleSpecifier,
referrer_kind: NodeModuleKind,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
if package_exports.contains_key(&package_subpath)
&& package_subpath.find('*').is_none()
@ -592,7 +592,7 @@ pub fn package_resolve(
referrer: &ModuleSpecifier,
referrer_kind: NodeModuleKind,
conditions: &[&str],
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
let (package_name, package_subpath, _is_scoped) =
parse_package_name(specifier, referrer)?;
@ -656,7 +656,7 @@ pub fn package_resolve(
pub fn get_package_scope_config(
referrer: &ModuleSpecifier,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PackageJson, AnyError> {
let root_folder = npm_resolver
.resolve_package_folder_from_path(&referrer.to_file_path().unwrap())?;
@ -666,7 +666,7 @@ pub fn get_package_scope_config(
pub fn get_closest_package_json(
url: &ModuleSpecifier,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PackageJson, AnyError> {
let package_json_path = get_closest_package_json_path(url, npm_resolver)?;
PackageJson::load(npm_resolver, package_json_path)
@ -674,7 +674,7 @@ pub fn get_closest_package_json(
fn get_closest_package_json_path(
url: &ModuleSpecifier,
npm_resolver: &dyn DenoDirNpmResolver,
npm_resolver: &dyn RequireNpmResolver,
) -> Result<PathBuf, AnyError> {
let file_path = url.to_file_path().unwrap();
let mut current_dir = file_path.parent().unwrap();

4
runtime/web_worker.rs
View file

@ -31,7 +31,7 @@ use deno_core::ModuleSpecifier;
use deno_core::RuntimeOptions;
use deno_core::SharedArrayBufferStore;
use deno_core::SourceMapGetter;
use deno_node::DenoDirNpmResolver;
use deno_node::RequireNpmResolver;
use deno_tls::rustls::RootCertStore;
use deno_web::create_entangled_message_port;
use deno_web::BlobStore;
@ -324,7 +324,7 @@ pub struct WebWorkerOptions {
pub root_cert_store: Option<RootCertStore>,
pub seed: Option<u64>,
pub module_loader: Rc<dyn ModuleLoader>,
pub npm_resolver: Option<Rc<dyn DenoDirNpmResolver>>,
pub npm_resolver: Option<Rc<dyn RequireNpmResolver>>,
pub create_web_worker_cb: Arc<ops::worker_host::CreateWebWorkerCb>,
pub preload_module_cb: Arc<ops::worker_host::WorkerEventCb>,
pub pre_execute_module_cb: Arc<ops::worker_host::WorkerEventCb>,

4
runtime/worker.rs
View file

@ -25,7 +25,7 @@ use deno_core::ModuleSpecifier;
use deno_core::RuntimeOptions;
use deno_core::SharedArrayBufferStore;
use deno_core::SourceMapGetter;
use deno_node::DenoDirNpmResolver;
use deno_node::RequireNpmResolver;
use deno_tls::rustls::RootCertStore;
use deno_web::BlobStore;
use log::debug;
@ -75,7 +75,7 @@ pub struct WorkerOptions {
pub root_cert_store: Option<RootCertStore>,
pub seed: Option<u64>,
pub module_loader: Rc<dyn ModuleLoader>,
pub npm_resolver: Option<Rc<dyn DenoDirNpmResolver>>,
pub npm_resolver: Option<Rc<dyn RequireNpmResolver>>,
// Callbacks invoked when creating new instance of WebWorker
pub create_web_worker_cb: Arc<ops::worker_host::CreateWebWorkerCb>,
pub web_worker_preload_module_cb: Arc<ops::worker_host::WorkerEventCb>,