mirror of
https://github.com/golang/go
synced 2024-09-05 16:04:50 +00:00
cmd/go: parallelize matchPackages work in each module
In each module matchPackages looks in, when doing the walk, do the scanDir call in a par.Queue so all the read work can be done in parallel. Change-Id: I27153dbb3a2ed417ca24972f47134e9e914a55d1 Reviewed-on: https://go-review.googlesource.com/c/go/+/404097 Reviewed-by: Michael Matloob <matloob@golang.org> Run-TryBot: Michael Matloob <matloob@golang.org> TryBot-Result: Gopher Robot <gobot@golang.org> Reviewed-by: Bryan Mills <bcmills@google.com>
This commit is contained in:
parent
1d9d99b7ce
commit
74f1fa6ecb
|
@ -12,12 +12,16 @@ import (
|
|||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"cmd/go/internal/cfg"
|
||||
"cmd/go/internal/fsys"
|
||||
"cmd/go/internal/imports"
|
||||
"cmd/go/internal/modindex"
|
||||
"cmd/go/internal/par"
|
||||
"cmd/go/internal/search"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
|
@ -43,9 +47,15 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
|
||||
}
|
||||
|
||||
var mu sync.Mutex
|
||||
have := map[string]bool{
|
||||
"builtin": true, // ignore pseudo-package that exists only for documentation
|
||||
}
|
||||
addPkg := func(p string) {
|
||||
mu.Lock()
|
||||
m.Pkgs = append(m.Pkgs, p)
|
||||
mu.Unlock()
|
||||
}
|
||||
if !cfg.BuildContext.CgoEnabled {
|
||||
have["runtime/cgo"] = true // ignore during walk
|
||||
}
|
||||
|
@ -56,6 +66,8 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
pruneGoMod
|
||||
)
|
||||
|
||||
q := par.NewQueue(runtime.GOMAXPROCS(0))
|
||||
|
||||
walkPkgs := func(root, importPathRoot string, prune pruning) {
|
||||
root = filepath.Clean(root)
|
||||
err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
|
||||
|
@ -110,9 +122,11 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
if !have[name] {
|
||||
have[name] = true
|
||||
if isMatch(name) {
|
||||
if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
|
||||
m.Pkgs = append(m.Pkgs, name)
|
||||
}
|
||||
q.Add(func() {
|
||||
if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
|
||||
addPkg(name)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -126,6 +140,12 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
}
|
||||
}
|
||||
|
||||
// Wait for all in-flight operations to complete before returning.
|
||||
defer func() {
|
||||
<-q.Idle()
|
||||
sort.Strings(m.Pkgs) // sort everything we added for determinism
|
||||
}()
|
||||
|
||||
if filter == includeStd {
|
||||
walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
|
||||
if treeCanMatch("cmd") {
|
||||
|
@ -169,7 +189,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
modPrefix = mod.Path
|
||||
}
|
||||
if mi, err := modindex.Get(root); err == nil {
|
||||
walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
|
||||
walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
|
||||
continue
|
||||
} else if !errors.Is(err, modindex.ErrNotIndexed) {
|
||||
m.AddError(err)
|
||||
|
@ -188,13 +208,7 @@ func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, f
|
|||
// walkFromIndex matches packages in a module using the module index. modroot
|
||||
// is the module's root directory on disk, index is the ModuleIndex for the
|
||||
// module, and importPathRoot is the module's path prefix.
|
||||
func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
|
||||
isMatch := func(string) bool { return true }
|
||||
treeCanMatch := func(string) bool { return true }
|
||||
if !m.IsMeta() {
|
||||
isMatch = search.MatchPattern(m.Pattern())
|
||||
treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
|
||||
}
|
||||
func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
|
||||
loopPackages:
|
||||
for _, reldir := range index.Packages() {
|
||||
// Avoid .foo, _foo, and testdata subdirectory trees.
|
||||
|
@ -232,7 +246,7 @@ loopPackages:
|
|||
have[name] = true
|
||||
if isMatch(name) {
|
||||
if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
|
||||
m.Pkgs = append(m.Pkgs, name)
|
||||
addPkg(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue