languages/go
languages/go
1
0
Fork 0
mirror of https://github.com/golang/go synced 2024-09-05 16:04:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

cmd/go: parallelize matchPackages work in each module

Browse source 
In each module matchPackages looks in, when doing the walk, do the
scanDir call in a par.Queue so all the read work can be done in
parallel.

Change-Id: I27153dbb3a2ed417ca24972f47134e9e914a55d1
Reviewed-on: https://go-review.googlesource.com/c/go/+/404097
Reviewed-by: Michael Matloob <matloob@golang.org>
Run-TryBot: Michael Matloob <matloob@golang.org>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Bryan Mills <bcmills@google.com>
This commit is contained in:
Michael Matloob 2022-05-04 17:11:35 -04:00
parent 1d9d99b7ce
commit 74f1fa6ecb
1 changed files with 26 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
src/cmd/go/internal/modload/search.go
View file

@ -12,12 +12,16 @@ import (
"os"
"path"
"path/filepath"
"runtime"
"sort"
"strings"
"sync"
"cmd/go/internal/cfg"
"cmd/go/internal/fsys"
"cmd/go/internal/imports"
"cmd/go/internal/modindex"
"cmd/go/internal/par"
"cmd/go/internal/search"
"golang.org/x/mod/module"
@ -43,9 +47,15 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
}
var mu sync.Mutex
have := map[string]bool{
"builtin": true, // ignore pseudo-package that exists only for documentation
}
addPkg := func(p string) {
mu.Lock()
m.Pkgs = append(m.Pkgs, p)
mu.Unlock()
}
if !cfg.BuildContext.CgoEnabled {
have["runtime/cgo"] = true // ignore during walk
}
@ -56,6 +66,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
pruneGoMod
)
q := par.NewQueue(runtime.GOMAXPROCS(0))
walkPkgs := func(root, importPathRoot string, prune pruning) {
root = filepath.Clean(root)
err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
@ -110,9 +122,11 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
if !have[name] {
have[name] = true
if isMatch(name) {
if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
m.Pkgs = append(m.Pkgs, name)
}
q.Add(func() {
if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
addPkg(name)
}
})
}
}
@ -126,6 +140,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
}
}
// Wait for all in-flight operations to complete before returning.
defer func() {
<-q.Idle()
sort.Strings(m.Pkgs) // sort everything we added for determinism
}()
if filter == includeStd {
walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
if treeCanMatch("cmd") {
@ -169,7 +189,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
modPrefix = mod.Path
}
if mi, err := modindex.Get(root); err == nil {
walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
continue
} else if !errors.Is(err, modindex.ErrNotIndexed) {
m.AddError(err)
@ -188,13 +208,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
// walkFromIndex matches packages in a module using the module index. modroot
// is the module's root directory on disk, index is the ModuleIndex for the
// module, and importPathRoot is the module's path prefix.
func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
isMatch := func(string) bool { return true }
treeCanMatch := func(string) bool { return true }
if !m.IsMeta() {
isMatch = search.MatchPattern(m.Pattern())
treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
}
func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
loopPackages:
for _, reldir := range index.Packages() {
// Avoid .foo, _foo, and testdata subdirectory trees.
@ -232,7 +246,7 @@ loopPackages:
have[name] = true
if isMatch(name) {
if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
m.Pkgs = append(m.Pkgs, name)
addPkg(name)
}
}
}