// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; use deno_lockfile::NpmPackageDependencyLockfileInfo; use deno_lockfile::NpmPackageLockfileInfo; use deno_npm::registry::NpmRegistryApi; use deno_npm::resolution::NpmPackagesPartitioned; use deno_npm::resolution::NpmResolutionError; use deno_npm::resolution::NpmResolutionSnapshot; use deno_npm::resolution::NpmResolutionSnapshotPendingResolver; use deno_npm::resolution::NpmResolutionSnapshotPendingResolverOptions; use deno_npm::resolution::PackageCacheFolderIdNotFoundError; use deno_npm::resolution::PackageNotFoundFromReferrerError; use deno_npm::resolution::PackageNvNotFoundError; use deno_npm::resolution::PackageReqNotFoundError; use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; use deno_semver::package::PackageNv; use deno_semver::package::PackageReq; use deno_semver::VersionReq; use crate::args::Lockfile; use crate::util::sync::SyncReadAsyncWriteLock; use super::CliNpmRegistryApi; pub struct AddPkgReqsResult { /// Results from adding the individual packages. /// /// The indexes of the results correspond to the indexes of the provided /// package requirements. pub results: Vec>, /// The final result of resolving and caching all the package requirements. pub dependencies_result: Result<(), AnyError>, } /// Handles updating and storing npm resolution in memory where the underlying /// snapshot can be updated concurrently. Additionally handles updating the lockfile /// based on changes to the resolution. /// /// This does not interact with the file system. pub struct NpmResolution { api: Arc, snapshot: SyncReadAsyncWriteLock, maybe_lockfile: Option>>, } impl std::fmt::Debug for NpmResolution { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snapshot = self.snapshot.read(); f.debug_struct("NpmResolution") .field("snapshot", &snapshot.as_valid_serialized().as_serialized()) .finish() } } impl NpmResolution { pub fn from_serialized( api: Arc, initial_snapshot: Option, maybe_lockfile: Option>>, ) -> Self { let snapshot = NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default()); Self::new(api, snapshot, maybe_lockfile) } pub fn new( api: Arc, initial_snapshot: NpmResolutionSnapshot, maybe_lockfile: Option>>, ) -> Self { Self { api, snapshot: SyncReadAsyncWriteLock::new(initial_snapshot), maybe_lockfile, } } pub async fn add_package_reqs( &self, package_reqs: &[PackageReq], ) -> AddPkgReqsResult { // only allow one thread in here at a time let snapshot_lock = self.snapshot.acquire().await; let result = add_package_reqs_to_snapshot( &self.api, package_reqs, self.maybe_lockfile.clone(), || snapshot_lock.read().clone(), ) .await; AddPkgReqsResult { results: result.results, dependencies_result: match result.dep_graph_result { Ok(snapshot) => { *snapshot_lock.write() = snapshot; Ok(()) } Err(err) => Err(err.into()), }, } } pub async fn set_package_reqs( &self, package_reqs: &[PackageReq], ) -> Result<(), AnyError> { // only allow one thread in here at a time let snapshot_lock = self.snapshot.acquire().await; let reqs_set = package_reqs.iter().collect::>(); let snapshot = add_package_reqs_to_snapshot( &self.api, package_reqs, self.maybe_lockfile.clone(), || { let snapshot = snapshot_lock.read().clone(); let has_removed_package = !snapshot .package_reqs() .keys() .all(|req| reqs_set.contains(req)); // if any packages were removed, we need to completely recreate the npm resolution snapshot if has_removed_package { snapshot.into_empty() } else { snapshot } }, ) .await .into_result()?; *snapshot_lock.write() = snapshot; Ok(()) } pub fn resolve_pkg_cache_folder_id_from_pkg_id( &self, id: &NpmPackageId, ) -> Option { self .snapshot .read() .package_from_id(id) .map(|p| p.get_package_cache_folder_id()) } pub fn resolve_pkg_id_from_pkg_cache_folder_id( &self, id: &NpmPackageCacheFolderId, ) -> Result { self .snapshot .read() .resolve_pkg_from_pkg_cache_folder_id(id) .map(|pkg| pkg.id.clone()) } pub fn resolve_package_from_package( &self, name: &str, referrer: &NpmPackageCacheFolderId, ) -> Result> { self .snapshot .read() .resolve_package_from_package(name, referrer) .cloned() } /// Resolve a node package from a deno module. pub fn resolve_pkg_id_from_pkg_req( &self, req: &PackageReq, ) -> Result { self .snapshot .read() .resolve_pkg_from_pkg_req(req) .map(|pkg| pkg.id.clone()) } pub fn resolve_pkg_reqs_from_pkg_id( &self, id: &NpmPackageId, ) -> Vec { let snapshot = self.snapshot.read(); let mut pkg_reqs = snapshot .package_reqs() .iter() .filter(|(_, nv)| *nv == &id.nv) .map(|(req, _)| req.clone()) .collect::>(); pkg_reqs.sort(); // be deterministic pkg_reqs } pub fn resolve_pkg_id_from_deno_module( &self, id: &PackageNv, ) -> Result { self .snapshot .read() .resolve_package_from_deno_module(id) .map(|pkg| pkg.id.clone()) } pub fn package_reqs(&self) -> HashMap { self.snapshot.read().package_reqs().clone() } pub fn all_system_packages( &self, system_info: &NpmSystemInfo, ) -> Vec { self.snapshot.read().all_system_packages(system_info) } pub fn all_system_packages_partitioned( &self, system_info: &NpmSystemInfo, ) -> NpmPackagesPartitioned { self .snapshot .read() .all_system_packages_partitioned(system_info) } pub fn snapshot(&self) -> NpmResolutionSnapshot { self.snapshot.read().clone() } pub fn serialized_valid_snapshot( &self, ) -> ValidSerializedNpmResolutionSnapshot { self.snapshot.read().as_valid_serialized() } pub fn serialized_valid_snapshot_for_system( &self, system_info: &NpmSystemInfo, ) -> ValidSerializedNpmResolutionSnapshot { self .snapshot .read() .as_valid_serialized_for_system(system_info) } } async fn add_package_reqs_to_snapshot( api: &CliNpmRegistryApi, package_reqs: &[PackageReq], maybe_lockfile: Option>>, get_new_snapshot: impl Fn() -> NpmResolutionSnapshot, ) -> deno_npm::resolution::AddPkgReqsResult { let snapshot = get_new_snapshot(); if package_reqs .iter() .all(|req| snapshot.package_reqs().contains_key(req)) { log::debug!("Snapshot already up to date. Skipping npm resolution."); return deno_npm::resolution::AddPkgReqsResult { results: package_reqs .iter() .map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone())) .collect(), dep_graph_result: Ok(snapshot), }; } log::debug!( /* this string is used in tests */ "Running npm resolution." ); let pending_resolver = get_npm_pending_resolver(api); let result = pending_resolver.add_pkg_reqs(snapshot, package_reqs).await; api.clear_memory_cache(); let result = match &result.dep_graph_result { Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => { log::debug!("{err:#}"); log::debug!("npm resolution failed. Trying again..."); // try again let snapshot = get_new_snapshot(); let result = pending_resolver.add_pkg_reqs(snapshot, package_reqs).await; api.clear_memory_cache(); result } _ => result, }; if let Ok(snapshot) = &result.dep_graph_result { if let Some(lockfile_mutex) = maybe_lockfile { let mut lockfile = lockfile_mutex.lock(); populate_lockfile_from_snapshot(&mut lockfile, snapshot); } } result } fn get_npm_pending_resolver( api: &CliNpmRegistryApi, ) -> NpmResolutionSnapshotPendingResolver { NpmResolutionSnapshotPendingResolver::new( NpmResolutionSnapshotPendingResolverOptions { api, // WARNING: When bumping this version, check if anything needs to be // updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js types_node_version_req: Some( VersionReq::parse_from_npm("18.0.0 - 18.16.19").unwrap(), ), }, ) } fn populate_lockfile_from_snapshot( lockfile: &mut Lockfile, snapshot: &NpmResolutionSnapshot, ) { for (package_req, nv) in snapshot.package_reqs() { lockfile.insert_package_specifier( format!("npm:{}", package_req), format!( "npm:{}", snapshot .resolve_package_from_deno_module(nv) .unwrap() .id .as_serialized() ), ); } for package in snapshot.all_packages_for_every_system() { lockfile.insert_npm_package(npm_package_to_lockfile_info(package)); } } fn npm_package_to_lockfile_info( pkg: &NpmResolutionPackage, ) -> NpmPackageLockfileInfo { let dependencies = pkg .dependencies .iter() .map(|(name, id)| NpmPackageDependencyLockfileInfo { name: name.clone(), id: id.as_serialized(), }) .collect(); NpmPackageLockfileInfo { serialized_id: pkg.id.as_serialized(), integrity: pkg.dist.integrity().for_lockfile(), dependencies, } }