refactor(lsp): collect npm reqs by scope (#24172)

cli/lsp/config.rs

@@ -778,13 +778,9 @@ impl Settings {
     specifier: &ModuleSpecifier,
   ) -> (&WorkspaceSettings, Option<&ModuleSpecifier>) {
     let Ok(path) = specifier_to_file_path(specifier) else {
-      return (&self.unscoped, None);
+      return (&self.unscoped, self.first_folder.as_ref());
     };
     for (folder_uri, settings) in self.by_workspace_folder.iter().rev() {
-      let mut settings = settings.as_ref();
-      if self.first_folder.as_ref() == Some(folder_uri) {
-        settings = settings.or(Some(&self.unscoped));
-      }
       if let Some(settings) = settings {
         let Ok(folder_path) = specifier_to_file_path(folder_uri) else {
           continue;
@@ -794,7 +790,7 @@ impl Settings {
         }
       }
     }
-    (&self.unscoped, None)
+    (&self.unscoped, self.first_folder.as_ref())
   }
 
   pub fn enable_settings_hash(&self) -> u64 {
@@ -1572,6 +1568,10 @@ pub struct ConfigTree {
 }
 
 impl ConfigTree {
+  pub fn root_scope(&self) -> Option<&ModuleSpecifier> {
+    self.first_folder.as_ref()
+  }
+
   pub fn root_data(&self) -> Option<&ConfigData> {
     self.first_folder.as_ref().and_then(|s| self.scopes.get(s))
   }
@@ -1583,10 +1583,6 @@ impl ConfigTree {
       .unwrap_or_default()
   }
 
-  pub fn root_lockfile(&self) -> Option<&Arc<Mutex<Lockfile>>> {
-    self.root_data().and_then(|d| d.lockfile.as_ref())
-  }
-
   pub fn root_import_map(&self) -> Option<&Arc<ImportMap>> {
     self.root_data().and_then(|d| d.import_map.as_ref())
   }

cli/lsp/documents.rs

@@ -34,6 +34,7 @@ use deno_semver::npm::NpmPackageReqReference;
 use deno_semver::package::PackageReq;
 use indexmap::IndexMap;
 use std::borrow::Cow;
+use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::collections::HashSet;
@@ -903,7 +904,8 @@ pub struct Documents {
   /// settings.
   resolver: Arc<LspResolver>,
   /// The npm package requirements found in npm specifiers.
-  npm_specifier_reqs: Arc<Vec<PackageReq>>,
+  npm_reqs_by_scope:
+    Arc<BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>>,
   /// Gets if any document had a node: specifier such that a @types/node package
   /// should be injected.
   has_injected_types_node_package: bool,
@@ -1093,10 +1095,11 @@ impl Documents {
     false
   }
 
-  /// Returns a collection of npm package requirements.
+  pub fn npm_reqs_by_scope(
-  pub fn npm_package_reqs(&mut self) -> Arc<Vec<PackageReq>> {
+    &mut self,
+  ) -> Arc<BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>> {
     self.calculate_npm_reqs_if_dirty();
-    self.npm_specifier_reqs.clone()
+    self.npm_reqs_by_scope.clone()
   }
 
   /// Returns if a @types/node package was injected into the npm
@@ -1322,31 +1325,36 @@ impl Documents {
   /// document and the value is a set of specifiers that depend on that
   /// document.
   fn calculate_npm_reqs_if_dirty(&mut self) {
-    let mut npm_reqs = HashSet::new();
+    let mut npm_reqs_by_scope: BTreeMap<_, BTreeSet<_>> = Default::default();
-    let mut has_node_builtin_specifier = false;
+    let mut scopes_with_node_builtin_specifier = HashSet::new();
     let is_fs_docs_dirty = self.file_system_docs.set_dirty(false);
     if !is_fs_docs_dirty && !self.dirty {
       return;
     }
     let mut visit_doc = |doc: &Arc<Document>| {
+      let scope = doc
+        .file_referrer()
+        .and_then(|r| self.config.tree.scope_for_specifier(r))
+        .or(self.config.tree.root_scope());
+      let reqs = npm_reqs_by_scope.entry(scope.cloned()).or_default();
       for dependency in doc.dependencies().values() {
         if let Some(dep) = dependency.get_code() {
           if dep.scheme() == "node" {
-            has_node_builtin_specifier = true;
+            scopes_with_node_builtin_specifier.insert(scope.cloned());
           }
           if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
-            npm_reqs.insert(reference.into_inner().req);
+            reqs.insert(reference.into_inner().req);
           }
         }
         if let Some(dep) = dependency.get_type() {
           if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
-            npm_reqs.insert(reference.into_inner().req);
+            reqs.insert(reference.into_inner().req);
           }
         }
       }
       if let Some(dep) = doc.maybe_types_dependency().maybe_specifier() {
         if let Ok(reference) = NpmPackageReqReference::from_specifier(dep) {
-          npm_reqs.insert(reference.into_inner().req);
+          reqs.insert(reference.into_inner().req);
         }
       }
     };
@@ -1358,12 +1366,21 @@ impl Documents {
     }
 
     // fill the reqs from the lockfile
-    if let Some(lockfile) = self.config.tree.root_lockfile() {
+    // TODO(nayeemrmn): Iterate every lockfile here for multi-deno.json.
+    if let Some(lockfile) = self
+      .config
+      .tree
+      .root_data()
+      .and_then(|d| d.lockfile.as_ref())
+    {
+      let reqs = npm_reqs_by_scope
+        .entry(self.config.tree.root_scope().cloned())
+        .or_default();
       let lockfile = lockfile.lock();
       for key in lockfile.content.packages.specifiers.keys() {
         if let Some(key) = key.strip_prefix("npm:") {
           if let Ok(req) = PackageReq::from_str(key) {
-            npm_reqs.insert(req);
+            reqs.insert(req);
           }
         }
       }
@@ -1372,17 +1389,15 @@ impl Documents {
     // Ensure a @types/node package exists when any module uses a node: specifier.
     // Unlike on the command line, here we just add @types/node to the npm package
     // requirements since this won't end up in the lockfile.
-    self.has_injected_types_node_package = has_node_builtin_specifier
+    for scope in scopes_with_node_builtin_specifier {
-      && !npm_reqs.iter().any(|r| r.name == "@types/node");
+      let reqs = npm_reqs_by_scope.entry(scope).or_default();
-    if self.has_injected_types_node_package {
+      if !reqs.iter().any(|r| r.name == "@types/node") {
-      npm_reqs.insert(PackageReq::from_str("@types/node").unwrap());
+        self.has_injected_types_node_package = true;
+        reqs.insert(PackageReq::from_str("@types/node").unwrap());
+      }
     }
 
-    self.npm_specifier_reqs = Arc::new({
+    self.npm_reqs_by_scope = Arc::new(npm_reqs_by_scope);
-      let mut reqs = npm_reqs.into_iter().collect::<Vec<_>>();
-      reqs.sort();
-      reqs
-    });
     self.dirty = false;
   }
 

cli/lsp/language_server.rs

@@ -806,7 +806,7 @@ impl Inner {
       log::debug!("Skipped workspace walk due to client incapability.");
       return (Default::default(), false);
     }
-    let mut workspace_files = Default::default();
+    let mut workspace_files = BTreeSet::default();
     let entry_limit = 1000;
     let mut pending = VecDeque::new();
     let mut entry_count = 0;
@@ -816,10 +816,30 @@ impl Inner {
       .filter_map(|p| specifier_to_file_path(&p.0).ok())
       .collect::<Vec<_>>();
     roots.sort();
-    for i in 0..roots.len() {
+    let roots = roots
-      if i == 0 || !roots[i].starts_with(&roots[i - 1]) {
+      .iter()
-        if let Ok(read_dir) = std::fs::read_dir(&roots[i]) {
+      .enumerate()
-          pending.push_back((roots[i].clone(), read_dir));
+      .filter(|(i, root)| *i == 0 || !root.starts_with(&roots[i - 1]))
+      .map(|(_, r)| r.clone())
+      .collect::<Vec<_>>();
+    let mut root_ancestors = BTreeSet::new();
+    for root in roots {
+      for ancestor in root.ancestors().skip(1) {
+        if root_ancestors.insert(ancestor.to_path_buf()) {
+          break;
+        }
+      }
+      if let Ok(read_dir) = std::fs::read_dir(&root) {
+        pending.push_back((root, read_dir));
+      }
+    }
+    for root_ancestor in root_ancestors {
+      for deno_json in ["deno.json", "deno.jsonc"] {
+        let path = root_ancestor.join(deno_json);
+        if path.exists() {
+          if let Ok(specifier) = ModuleSpecifier::from_file_path(path) {
+            workspace_files.insert(specifier);
+          }
         }
       }
     }
@@ -836,17 +856,15 @@ impl Inner {
         let Ok(specifier) = ModuleSpecifier::from_file_path(&path) else {
           continue;
         };
-        // TODO(nayeemrmn): Don't walk folders that are `None` here and aren't
-        // in a `deno.json` scope.
-        if config.settings.specifier_enabled(&specifier) == Some(false) {
-          continue;
-        }
         let Ok(file_type) = entry.file_type() else {
           continue;
         };
         let Some(file_name) = path.file_name() else {
           continue;
         };
+        if config.settings.specifier_enabled(&specifier) == Some(false) {
+          continue;
+        }
         if file_type.is_dir() {
           let dir_name = file_name.to_string_lossy().to_lowercase();
           // We ignore these directories by default because there is a
@@ -1107,11 +1125,11 @@ impl Inner {
   }
 
   async fn refresh_npm_specifiers(&mut self) {
-    let package_reqs = self.documents.npm_package_reqs();
+    let package_reqs = self.documents.npm_reqs_by_scope();
     let resolver = self.resolver.clone();
     // spawn due to the lsp's `Send` requirement
     let handle =
-      spawn(async move { resolver.set_npm_package_reqs(&package_reqs).await });
+      spawn(async move { resolver.set_npm_reqs(&package_reqs).await });
     if let Err(err) = handle.await.unwrap() {
       lsp_warn!("Could not set npm package requirements. {:#}", err);
     }
@@ -3428,7 +3446,10 @@ impl Inner {
     roots.extend(
       self
         .documents
-        .npm_package_reqs()
+        .npm_reqs_by_scope()
+        .values()
+        .flatten()
+        .collect::<BTreeSet<_>>()
         .iter()
         .map(|req| ModuleSpecifier::parse(&format!("npm:{}", req)).unwrap()),
     );
@@ -3714,6 +3735,7 @@ mod tests {
 
     temp_dir.create_dir_all("root2/folder");
     temp_dir.create_dir_all("root2/sub_folder");
+    temp_dir.create_dir_all("root2/root2.1");
     temp_dir.write("root2/file1.ts", ""); // yes, enabled
     temp_dir.write("root2/file2.ts", ""); // no, not enabled
     temp_dir.write("root2/folder/main.ts", ""); // yes, enabled
@@ -3721,14 +3743,21 @@ mod tests {
     temp_dir.write("root2/sub_folder/a.js", ""); // no, not enabled
     temp_dir.write("root2/sub_folder/b.ts", ""); // no, not enabled
     temp_dir.write("root2/sub_folder/c.js", ""); // no, not enabled
+    temp_dir.write("root2/root2.1/main.ts", ""); // yes, enabled as separate root
 
     temp_dir.create_dir_all("root3/");
     temp_dir.write("root3/mod.ts", ""); // no, not enabled
 
+    temp_dir.create_dir_all("root4_parent/root4");
+    temp_dir.write("root4_parent/deno.json", ""); // yes, enabled as deno.json above root
+    temp_dir.write("root4_parent/root4/main.ts", ""); // yes, enabled
+
     let mut config = Config::new_with_roots(vec![
       temp_dir.uri().join("root1/").unwrap(),
       temp_dir.uri().join("root2/").unwrap(),
+      temp_dir.uri().join("root2/root2.1/").unwrap(),
       temp_dir.uri().join("root3/").unwrap(),
+      temp_dir.uri().join("root4_parent/root4/").unwrap(),
     ]);
     config.set_client_capabilities(ClientCapabilities {
       workspace: Some(Default::default()),
@@ -3756,6 +3785,13 @@ mod tests {
             ..Default::default()
           },
         ),
+        (
+          temp_dir.uri().join("root2/root2.1/").unwrap(),
+          WorkspaceSettings {
+            enable: Some(true),
+            ..Default::default()
+          },
+        ),
         (
           temp_dir.uri().join("root3/").unwrap(),
           WorkspaceSettings {
@@ -3763,6 +3799,13 @@ mod tests {
             ..Default::default()
           },
         ),
+        (
+          temp_dir.uri().join("root4_parent/root4/").unwrap(),
+          WorkspaceSettings {
+            enable: Some(true),
+            ..Default::default()
+          },
+        ),
       ],
     );
 
@@ -3784,6 +3827,9 @@ mod tests {
         temp_dir.uri().join("root1/sub_dir/mod.ts").unwrap(),
         temp_dir.uri().join("root2/file1.ts").unwrap(),
         temp_dir.uri().join("root2/folder/main.ts").unwrap(),
+        temp_dir.uri().join("root2/root2.1/main.ts").unwrap(),
+        temp_dir.uri().join("root4_parent/deno.json").unwrap(),
+        temp_dir.uri().join("root4_parent/root4/main.ts").unwrap(),
       ])
     );
   }

cli/lsp/resolver.rs

@@ -43,6 +43,8 @@ use deno_semver::package::PackageReq;
 use indexmap::IndexMap;
 use package_json::PackageJsonDepsProvider;
 use std::borrow::Cow;
+use std::collections::BTreeMap;
+use std::collections::BTreeSet;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::rc::Rc;
@@ -161,13 +163,20 @@ impl LspResolver {
     self.jsr_resolver.as_ref().inspect(|r| r.did_cache());
   }
 
-  pub async fn set_npm_package_reqs(
+  pub async fn set_npm_reqs(
     &self,
-    reqs: &[PackageReq],
+    reqs: &BTreeMap<Option<ModuleSpecifier>, BTreeSet<PackageReq>>,
   ) -> Result<(), AnyError> {
+    let reqs = reqs
+      .values()
+      .flatten()
+      .collect::<BTreeSet<_>>()
+      .into_iter()
+      .cloned()
+      .collect::<Vec<_>>();
     if let Some(npm_resolver) = self.npm_resolver.as_ref() {
       if let Some(npm_resolver) = npm_resolver.as_managed() {
-        return npm_resolver.set_package_reqs(reqs).await;
+        return npm_resolver.set_package_reqs(&reqs).await;
       }
     }
     Ok(())