From 74f1fa6ecbf79c778fc18d2a6b563fbb94f4b740 Mon Sep 17 00:00:00 2001
From: Michael Matloob <matloob@golang.org>
Date: Wed, 4 May 2022 17:11:35 -0400
Subject: [PATCH] cmd/go: parallelize matchPackages work in each module

In each module matchPackages looks in, when doing the walk, do the
scanDir call in a par.Queue so all the read work can be done in
parallel.

Change-Id: I27153dbb3a2ed417ca24972f47134e9e914a55d1
Reviewed-on: https://go-review.googlesource.com/c/go/+/404097
Reviewed-by: Michael Matloob <matloob@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
---
 src/cmd/go/internal/modload/search.go | 38 ++++++++++++++++++---------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/cmd/go/internal/modload/search.go b/src/cmd/go/internal/modload/search.go
index 60c68860edf..4b90392d94d 100644
--- a/src/cmd/go/internal/modload/search.go
+++ b/src/cmd/go/internal/modload/search.go
@@ -12,12 +12,16 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
+	"sort"
 	"strings"
+	"sync"
 
 	"cmd/go/internal/cfg"
 	"cmd/go/internal/fsys"
 	"cmd/go/internal/imports"
 	"cmd/go/internal/modindex"
+	"cmd/go/internal/par"
 	"cmd/go/internal/search"
 
 	"golang.org/x/mod/module"
@@ -43,9 +47,15 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 		treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
 	}
 
+	var mu sync.Mutex
 	have := map[string]bool{
 		"builtin": true, // ignore pseudo-package that exists only for documentation
 	}
+	addPkg := func(p string) {
+		mu.Lock()
+		m.Pkgs = append(m.Pkgs, p)
+		mu.Unlock()
+	}
 	if !cfg.BuildContext.CgoEnabled {
 		have["runtime/cgo"] = true // ignore during walk
 	}
@@ -56,6 +66,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 		pruneGoMod
 	)
 
+	q := par.NewQueue(runtime.GOMAXPROCS(0))
+
 	walkPkgs := func(root, importPathRoot string, prune pruning) {
 		root = filepath.Clean(root)
 		err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
@@ -110,9 +122,11 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 			if !have[name] {
 				have[name] = true
 				if isMatch(name) {
-					if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
-						m.Pkgs = append(m.Pkgs, name)
-					}
+					q.Add(func() {
+						if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
+							addPkg(name)
+						}
+					})
 				}
 			}
 
@@ -126,6 +140,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 		}
 	}
 
+	// Wait for all in-flight operations to complete before returning.
+	defer func() {
+		<-q.Idle()
+		sort.Strings(m.Pkgs) // sort everything we added for determinism
+	}()
+
 	if filter == includeStd {
 		walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
 		if treeCanMatch("cmd") {
@@ -169,7 +189,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 			modPrefix = mod.Path
 		}
 		if mi, err := modindex.Get(root); err == nil {
-			walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
+			walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
 			continue
 		} else if !errors.Is(err, modindex.ErrNotIndexed) {
 			m.AddError(err)
@@ -188,13 +208,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
 // walkFromIndex matches packages in a module using the module index. modroot
 // is the module's root directory on disk, index is the ModuleIndex for the
 // module, and importPathRoot is the module's path prefix.
-func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
-	isMatch := func(string) bool { return true }
-	treeCanMatch := func(string) bool { return true }
-	if !m.IsMeta() {
-		isMatch = search.MatchPattern(m.Pattern())
-		treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
-	}
+func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
 loopPackages:
 	for _, reldir := range index.Packages() {
 		// Avoid .foo, _foo, and testdata subdirectory trees.
@@ -232,7 +246,7 @@ loopPackages:
 			have[name] = true
 			if isMatch(name) {
 				if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
-					m.Pkgs = append(m.Pkgs, name)
+					addPkg(name)
 				}
 			}
 		}