2024-01-01 19:58:21 +00:00
|
|
|
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
|
2023-04-06 22:46:44 +00:00
|
|
|
|
2023-07-26 21:23:07 +00:00
|
|
|
use std::collections::HashMap;
|
2023-04-06 22:46:44 +00:00
|
|
|
use std::collections::HashSet;
|
|
|
|
use std::sync::Arc;
|
|
|
|
|
|
|
|
use deno_core::error::AnyError;
|
|
|
|
use deno_core::parking_lot::Mutex;
|
|
|
|
use deno_lockfile::NpmPackageDependencyLockfileInfo;
|
|
|
|
use deno_lockfile::NpmPackageLockfileInfo;
|
2023-08-08 17:07:29 +00:00
|
|
|
use deno_npm::registry::NpmRegistryApi;
|
2023-04-06 22:46:44 +00:00
|
|
|
use deno_npm::resolution::NpmPackagesPartitioned;
|
2023-04-07 01:41:19 +00:00
|
|
|
use deno_npm::resolution::NpmResolutionError;
|
2023-04-06 22:46:44 +00:00
|
|
|
use deno_npm::resolution::NpmResolutionSnapshot;
|
2023-06-30 12:49:49 +00:00
|
|
|
use deno_npm::resolution::NpmResolutionSnapshotPendingResolver;
|
|
|
|
use deno_npm::resolution::NpmResolutionSnapshotPendingResolverOptions;
|
2023-07-02 01:07:57 +00:00
|
|
|
use deno_npm::resolution::PackageCacheFolderIdNotFoundError;
|
2023-04-07 01:41:19 +00:00
|
|
|
use deno_npm::resolution::PackageNotFoundFromReferrerError;
|
|
|
|
use deno_npm::resolution::PackageNvNotFoundError;
|
|
|
|
use deno_npm::resolution::PackageReqNotFoundError;
|
2023-04-13 14:47:45 +00:00
|
|
|
use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot;
|
2023-04-06 22:46:44 +00:00
|
|
|
use deno_npm::NpmPackageCacheFolderId;
|
|
|
|
use deno_npm::NpmPackageId;
|
|
|
|
use deno_npm::NpmResolutionPackage;
|
2023-05-17 21:38:50 +00:00
|
|
|
use deno_npm::NpmSystemInfo;
|
2023-08-21 09:53:52 +00:00
|
|
|
use deno_semver::package::PackageNv;
|
|
|
|
use deno_semver::package::PackageReq;
|
2023-04-13 14:47:45 +00:00
|
|
|
use deno_semver::VersionReq;
|
2023-04-06 22:46:44 +00:00
|
|
|
|
|
|
|
use crate::args::Lockfile;
|
2024-06-05 19:17:35 +00:00
|
|
|
use crate::util::sync::SyncReadAsyncWriteLock;
|
2023-04-06 22:46:44 +00:00
|
|
|
|
2023-10-02 21:53:55 +00:00
|
|
|
use super::CliNpmRegistryApi;
|
2023-04-06 22:46:44 +00:00
|
|
|
|
2024-06-11 12:55:12 +00:00
|
|
|
pub struct AddPkgReqsResult {
|
|
|
|
/// Results from adding the individual packages.
|
|
|
|
///
|
|
|
|
/// The indexes of the results correspond to the indexes of the provided
|
|
|
|
/// package requirements.
|
|
|
|
pub results: Vec<Result<PackageNv, NpmResolutionError>>,
|
|
|
|
/// The final result of resolving and caching all the package requirements.
|
|
|
|
pub dependencies_result: Result<(), AnyError>,
|
|
|
|
}
|
|
|
|
|
2023-04-06 22:46:44 +00:00
|
|
|
/// Handles updating and storing npm resolution in memory where the underlying
|
|
|
|
/// snapshot can be updated concurrently. Additionally handles updating the lockfile
|
|
|
|
/// based on changes to the resolution.
|
|
|
|
///
|
|
|
|
/// This does not interact with the file system.
|
2023-04-14 20:22:33 +00:00
|
|
|
pub struct NpmResolution {
|
|
|
|
api: Arc<CliNpmRegistryApi>,
|
2024-06-05 19:17:35 +00:00
|
|
|
snapshot: SyncReadAsyncWriteLock<NpmResolutionSnapshot>,
|
2023-04-06 22:46:44 +00:00
|
|
|
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl std::fmt::Debug for NpmResolution {
|
|
|
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
2023-04-14 20:22:33 +00:00
|
|
|
let snapshot = self.snapshot.read();
|
2023-04-06 22:46:44 +00:00
|
|
|
f.debug_struct("NpmResolution")
|
2023-06-08 15:48:29 +00:00
|
|
|
.field("snapshot", &snapshot.as_valid_serialized().as_serialized())
|
2023-04-06 22:46:44 +00:00
|
|
|
.finish()
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
impl NpmResolution {
|
2023-04-13 14:47:45 +00:00
|
|
|
pub fn from_serialized(
|
2023-04-14 20:22:33 +00:00
|
|
|
api: Arc<CliNpmRegistryApi>,
|
2023-04-13 14:47:45 +00:00
|
|
|
initial_snapshot: Option<ValidSerializedNpmResolutionSnapshot>,
|
|
|
|
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
|
|
|
|
) -> Self {
|
|
|
|
let snapshot =
|
2023-06-30 12:49:49 +00:00
|
|
|
NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default());
|
2023-04-13 14:47:45 +00:00
|
|
|
Self::new(api, snapshot, maybe_lockfile)
|
|
|
|
}
|
|
|
|
|
2023-04-06 22:46:44 +00:00
|
|
|
pub fn new(
|
2023-04-14 20:22:33 +00:00
|
|
|
api: Arc<CliNpmRegistryApi>,
|
2023-04-13 14:47:45 +00:00
|
|
|
initial_snapshot: NpmResolutionSnapshot,
|
2023-04-06 22:46:44 +00:00
|
|
|
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
|
|
|
|
) -> Self {
|
2023-04-14 20:22:33 +00:00
|
|
|
Self {
|
2023-04-06 22:46:44 +00:00
|
|
|
api,
|
2024-06-05 19:17:35 +00:00
|
|
|
snapshot: SyncReadAsyncWriteLock::new(initial_snapshot),
|
2023-04-06 22:46:44 +00:00
|
|
|
maybe_lockfile,
|
2023-04-14 20:22:33 +00:00
|
|
|
}
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub async fn add_package_reqs(
|
|
|
|
&self,
|
2023-08-21 09:53:52 +00:00
|
|
|
package_reqs: &[PackageReq],
|
2024-06-11 12:55:12 +00:00
|
|
|
) -> AddPkgReqsResult {
|
2023-04-06 22:46:44 +00:00
|
|
|
// only allow one thread in here at a time
|
2024-06-05 19:17:35 +00:00
|
|
|
let snapshot_lock = self.snapshot.acquire().await;
|
2024-06-11 12:55:12 +00:00
|
|
|
let result = add_package_reqs_to_snapshot(
|
2023-04-14 20:22:33 +00:00
|
|
|
&self.api,
|
2023-04-06 22:46:44 +00:00
|
|
|
package_reqs,
|
2023-04-14 20:22:33 +00:00
|
|
|
self.maybe_lockfile.clone(),
|
2024-06-05 19:17:35 +00:00
|
|
|
|| snapshot_lock.read().clone(),
|
2023-04-06 22:46:44 +00:00
|
|
|
)
|
2024-06-11 12:55:12 +00:00
|
|
|
.await;
|
|
|
|
|
|
|
|
AddPkgReqsResult {
|
|
|
|
results: result.results,
|
|
|
|
dependencies_result: match result.dep_graph_result {
|
|
|
|
Ok(snapshot) => {
|
|
|
|
*snapshot_lock.write() = snapshot;
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
Err(err) => Err(err.into()),
|
|
|
|
},
|
|
|
|
}
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub async fn set_package_reqs(
|
|
|
|
&self,
|
2023-08-21 09:53:52 +00:00
|
|
|
package_reqs: &[PackageReq],
|
2023-04-06 22:46:44 +00:00
|
|
|
) -> Result<(), AnyError> {
|
|
|
|
// only allow one thread in here at a time
|
2024-06-05 19:17:35 +00:00
|
|
|
let snapshot_lock = self.snapshot.acquire().await;
|
2023-04-07 01:41:19 +00:00
|
|
|
|
2023-05-22 20:55:04 +00:00
|
|
|
let reqs_set = package_reqs.iter().collect::<HashSet<_>>();
|
2023-04-06 22:46:44 +00:00
|
|
|
let snapshot = add_package_reqs_to_snapshot(
|
2023-04-14 20:22:33 +00:00
|
|
|
&self.api,
|
2023-04-06 22:46:44 +00:00
|
|
|
package_reqs,
|
2023-04-14 20:22:33 +00:00
|
|
|
self.maybe_lockfile.clone(),
|
2023-04-07 01:41:19 +00:00
|
|
|
|| {
|
2024-06-05 19:17:35 +00:00
|
|
|
let snapshot = snapshot_lock.read().clone();
|
2023-04-07 01:41:19 +00:00
|
|
|
let has_removed_package = !snapshot
|
|
|
|
.package_reqs()
|
|
|
|
.keys()
|
|
|
|
.all(|req| reqs_set.contains(req));
|
|
|
|
// if any packages were removed, we need to completely recreate the npm resolution snapshot
|
|
|
|
if has_removed_package {
|
2023-04-13 14:47:45 +00:00
|
|
|
snapshot.into_empty()
|
2023-04-07 01:41:19 +00:00
|
|
|
} else {
|
|
|
|
snapshot
|
|
|
|
}
|
|
|
|
},
|
2023-04-06 22:46:44 +00:00
|
|
|
)
|
2024-06-11 12:55:12 +00:00
|
|
|
.await
|
|
|
|
.into_result()?;
|
2023-04-06 22:46:44 +00:00
|
|
|
|
2024-06-05 19:17:35 +00:00
|
|
|
*snapshot_lock.write() = snapshot;
|
2023-04-06 22:46:44 +00:00
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
|
|
|
|
2023-07-02 01:07:57 +00:00
|
|
|
pub fn resolve_pkg_cache_folder_id_from_pkg_id(
|
2023-04-06 22:46:44 +00:00
|
|
|
&self,
|
|
|
|
id: &NpmPackageId,
|
|
|
|
) -> Option<NpmPackageCacheFolderId> {
|
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.package_from_id(id)
|
|
|
|
.map(|p| p.get_package_cache_folder_id())
|
|
|
|
}
|
|
|
|
|
2023-07-02 01:07:57 +00:00
|
|
|
pub fn resolve_pkg_id_from_pkg_cache_folder_id(
|
|
|
|
&self,
|
|
|
|
id: &NpmPackageCacheFolderId,
|
|
|
|
) -> Result<NpmPackageId, PackageCacheFolderIdNotFoundError> {
|
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.resolve_pkg_from_pkg_cache_folder_id(id)
|
|
|
|
.map(|pkg| pkg.id.clone())
|
|
|
|
}
|
|
|
|
|
2023-04-06 22:46:44 +00:00
|
|
|
pub fn resolve_package_from_package(
|
|
|
|
&self,
|
|
|
|
name: &str,
|
|
|
|
referrer: &NpmPackageCacheFolderId,
|
2023-04-07 01:41:19 +00:00
|
|
|
) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> {
|
2023-04-06 22:46:44 +00:00
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.resolve_package_from_package(name, referrer)
|
|
|
|
.cloned()
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Resolve a node package from a deno module.
|
|
|
|
pub fn resolve_pkg_id_from_pkg_req(
|
|
|
|
&self,
|
2023-08-21 09:53:52 +00:00
|
|
|
req: &PackageReq,
|
2023-04-07 01:41:19 +00:00
|
|
|
) -> Result<NpmPackageId, PackageReqNotFoundError> {
|
2023-04-06 22:46:44 +00:00
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.resolve_pkg_from_pkg_req(req)
|
2023-05-24 20:23:10 +00:00
|
|
|
.map(|pkg| pkg.id.clone())
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
2023-07-02 01:07:57 +00:00
|
|
|
pub fn resolve_pkg_reqs_from_pkg_id(
|
|
|
|
&self,
|
|
|
|
id: &NpmPackageId,
|
2023-08-21 09:53:52 +00:00
|
|
|
) -> Vec<PackageReq> {
|
2023-07-02 01:07:57 +00:00
|
|
|
let snapshot = self.snapshot.read();
|
|
|
|
let mut pkg_reqs = snapshot
|
|
|
|
.package_reqs()
|
|
|
|
.iter()
|
|
|
|
.filter(|(_, nv)| *nv == &id.nv)
|
|
|
|
.map(|(req, _)| req.clone())
|
|
|
|
.collect::<Vec<_>>();
|
|
|
|
pkg_reqs.sort(); // be deterministic
|
|
|
|
pkg_reqs
|
|
|
|
}
|
|
|
|
|
2023-04-06 22:46:44 +00:00
|
|
|
pub fn resolve_pkg_id_from_deno_module(
|
|
|
|
&self,
|
2023-08-21 09:53:52 +00:00
|
|
|
id: &PackageNv,
|
2023-04-07 01:41:19 +00:00
|
|
|
) -> Result<NpmPackageId, PackageNvNotFoundError> {
|
2023-04-06 22:46:44 +00:00
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.resolve_package_from_deno_module(id)
|
2023-05-24 20:23:10 +00:00
|
|
|
.map(|pkg| pkg.id.clone())
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
2023-08-21 09:53:52 +00:00
|
|
|
pub fn package_reqs(&self) -> HashMap<PackageReq, PackageNv> {
|
2023-07-26 21:23:07 +00:00
|
|
|
self.snapshot.read().package_reqs().clone()
|
|
|
|
}
|
|
|
|
|
2023-05-17 21:38:50 +00:00
|
|
|
pub fn all_system_packages(
|
|
|
|
&self,
|
|
|
|
system_info: &NpmSystemInfo,
|
|
|
|
) -> Vec<NpmResolutionPackage> {
|
|
|
|
self.snapshot.read().all_system_packages(system_info)
|
2023-05-11 00:06:59 +00:00
|
|
|
}
|
|
|
|
|
2023-05-17 21:38:50 +00:00
|
|
|
pub fn all_system_packages_partitioned(
|
|
|
|
&self,
|
|
|
|
system_info: &NpmSystemInfo,
|
|
|
|
) -> NpmPackagesPartitioned {
|
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.all_system_packages_partitioned(system_info)
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn snapshot(&self) -> NpmResolutionSnapshot {
|
2023-04-14 20:22:33 +00:00
|
|
|
self.snapshot.read().clone()
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
2023-06-08 15:48:29 +00:00
|
|
|
pub fn serialized_valid_snapshot(
|
|
|
|
&self,
|
|
|
|
) -> ValidSerializedNpmResolutionSnapshot {
|
|
|
|
self.snapshot.read().as_valid_serialized()
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn serialized_valid_snapshot_for_system(
|
|
|
|
&self,
|
|
|
|
system_info: &NpmSystemInfo,
|
|
|
|
) -> ValidSerializedNpmResolutionSnapshot {
|
|
|
|
self
|
|
|
|
.snapshot
|
|
|
|
.read()
|
|
|
|
.as_valid_serialized_for_system(system_info)
|
2023-04-13 14:47:45 +00:00
|
|
|
}
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
async fn add_package_reqs_to_snapshot(
|
2023-04-12 12:36:11 +00:00
|
|
|
api: &CliNpmRegistryApi,
|
2023-08-21 09:53:52 +00:00
|
|
|
package_reqs: &[PackageReq],
|
2023-04-06 22:46:44 +00:00
|
|
|
maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
|
2023-04-07 01:41:19 +00:00
|
|
|
get_new_snapshot: impl Fn() -> NpmResolutionSnapshot,
|
2024-06-11 12:55:12 +00:00
|
|
|
) -> deno_npm::resolution::AddPkgReqsResult {
|
2023-04-07 01:41:19 +00:00
|
|
|
let snapshot = get_new_snapshot();
|
2024-06-11 12:55:12 +00:00
|
|
|
if package_reqs
|
|
|
|
.iter()
|
|
|
|
.all(|req| snapshot.package_reqs().contains_key(req))
|
2023-04-06 22:46:44 +00:00
|
|
|
{
|
2024-06-11 12:55:12 +00:00
|
|
|
log::debug!("Snapshot already up to date. Skipping npm resolution.");
|
|
|
|
return deno_npm::resolution::AddPkgReqsResult {
|
|
|
|
results: package_reqs
|
|
|
|
.iter()
|
|
|
|
.map(|req| Ok(snapshot.package_reqs().get(req).unwrap().clone()))
|
|
|
|
.collect(),
|
|
|
|
dep_graph_result: Ok(snapshot),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
log::debug!(
|
|
|
|
/* this string is used in tests */
|
|
|
|
"Running npm resolution."
|
|
|
|
);
|
|
|
|
let pending_resolver = get_npm_pending_resolver(api);
|
|
|
|
let result = pending_resolver.add_pkg_reqs(snapshot, package_reqs).await;
|
|
|
|
api.clear_memory_cache();
|
|
|
|
let result = match &result.dep_graph_result {
|
|
|
|
Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => {
|
|
|
|
log::debug!("{err:#}");
|
|
|
|
log::debug!("npm resolution failed. Trying again...");
|
|
|
|
|
|
|
|
// try again
|
|
|
|
let snapshot = get_new_snapshot();
|
|
|
|
let result = pending_resolver.add_pkg_reqs(snapshot, package_reqs).await;
|
|
|
|
api.clear_memory_cache();
|
|
|
|
result
|
2023-04-07 01:41:19 +00:00
|
|
|
}
|
2024-06-11 12:55:12 +00:00
|
|
|
_ => result,
|
2023-04-07 01:41:19 +00:00
|
|
|
};
|
2023-04-06 22:46:44 +00:00
|
|
|
|
2024-06-11 12:55:12 +00:00
|
|
|
if let Ok(snapshot) = &result.dep_graph_result {
|
|
|
|
if let Some(lockfile_mutex) = maybe_lockfile {
|
|
|
|
let mut lockfile = lockfile_mutex.lock();
|
|
|
|
populate_lockfile_from_snapshot(&mut lockfile, snapshot);
|
|
|
|
}
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
2023-07-26 21:23:07 +00:00
|
|
|
|
2024-06-11 12:55:12 +00:00
|
|
|
result
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
|
2023-06-30 12:49:49 +00:00
|
|
|
fn get_npm_pending_resolver(
|
|
|
|
api: &CliNpmRegistryApi,
|
|
|
|
) -> NpmResolutionSnapshotPendingResolver<CliNpmRegistryApi> {
|
|
|
|
NpmResolutionSnapshotPendingResolver::new(
|
|
|
|
NpmResolutionSnapshotPendingResolverOptions {
|
|
|
|
api,
|
|
|
|
// WARNING: When bumping this version, check if anything needs to be
|
|
|
|
// updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js
|
|
|
|
types_node_version_req: Some(
|
2023-07-04 15:27:04 +00:00
|
|
|
VersionReq::parse_from_npm("18.0.0 - 18.16.19").unwrap(),
|
2023-06-30 12:49:49 +00:00
|
|
|
),
|
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2023-04-06 22:46:44 +00:00
|
|
|
fn populate_lockfile_from_snapshot(
|
|
|
|
lockfile: &mut Lockfile,
|
|
|
|
snapshot: &NpmResolutionSnapshot,
|
2024-05-28 18:58:43 +00:00
|
|
|
) {
|
2023-04-06 22:46:44 +00:00
|
|
|
for (package_req, nv) in snapshot.package_reqs() {
|
2023-09-08 18:34:57 +00:00
|
|
|
lockfile.insert_package_specifier(
|
|
|
|
format!("npm:{}", package_req),
|
|
|
|
format!(
|
|
|
|
"npm:{}",
|
|
|
|
snapshot
|
|
|
|
.resolve_package_from_deno_module(nv)
|
|
|
|
.unwrap()
|
|
|
|
.id
|
|
|
|
.as_serialized()
|
|
|
|
),
|
2023-04-06 22:46:44 +00:00
|
|
|
);
|
|
|
|
}
|
2023-05-17 21:38:50 +00:00
|
|
|
for package in snapshot.all_packages_for_every_system() {
|
2024-05-28 18:58:43 +00:00
|
|
|
lockfile.insert_npm_package(npm_package_to_lockfile_info(package));
|
2023-04-06 22:46:44 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
fn npm_package_to_lockfile_info(
|
2023-05-17 21:38:50 +00:00
|
|
|
pkg: &NpmResolutionPackage,
|
2023-04-06 22:46:44 +00:00
|
|
|
) -> NpmPackageLockfileInfo {
|
|
|
|
let dependencies = pkg
|
|
|
|
.dependencies
|
2023-05-17 21:38:50 +00:00
|
|
|
.iter()
|
2023-04-06 22:46:44 +00:00
|
|
|
.map(|(name, id)| NpmPackageDependencyLockfileInfo {
|
2023-05-17 21:38:50 +00:00
|
|
|
name: name.clone(),
|
2023-04-06 22:46:44 +00:00
|
|
|
id: id.as_serialized(),
|
|
|
|
})
|
|
|
|
.collect();
|
|
|
|
|
|
|
|
NpmPackageLockfileInfo {
|
2023-05-24 20:23:10 +00:00
|
|
|
serialized_id: pkg.id.as_serialized(),
|
2024-01-22 21:31:12 +00:00
|
|
|
integrity: pkg.dist.integrity().for_lockfile(),
|
2023-04-06 22:46:44 +00:00
|
|
|
dependencies,
|
|
|
|
}
|
|
|
|
}
|