fix(new_engine): respect --needed on target gathering (fixes #1552) (#1920)

* use logger in dep graph

* use logger in dep graph

* use logger in dep graph

* only query for AUR packages once per tier. useful for rpc

* fix performance regression for ros-melodic

* prefer name search first

* implement needed at target gathering

* set default config

* fixup tests for needed

go.mod

@@ -1,7 +1,7 @@
 module github.com/Jguer/yay/v11
 
 require (
-	github.com/Jguer/aur v1.2.1
+	github.com/Jguer/aur v1.2.3
 	github.com/Jguer/go-alpm/v2 v2.1.2
 	github.com/Jguer/votar v1.0.0
 	github.com/Morganamilo/go-pacmanconf v0.0.0-20210502114700-cff030e927a5

go.sum

@@ -1,5 +1,5 @@
-github.com/Jguer/aur v1.2.1 h1:7ViPn4EN10iMAhtUSugxytloCynyVI12HzfhnGG1xGE=
+github.com/Jguer/aur v1.2.3 h1:D+OGgLxnAnZnw88DsRvnRQsn0Poxsy9ng7pBcsA0krM=
-github.com/Jguer/aur v1.2.1/go.mod h1:Dahvb6L1yr0rR7svyYSDwaRJoQMeyvJblwJ3QH/7CUs=
+github.com/Jguer/aur v1.2.3/go.mod h1:Dahvb6L1yr0rR7svyYSDwaRJoQMeyvJblwJ3QH/7CUs=
 github.com/Jguer/go-alpm/v2 v2.1.2 h1:CGTIxzuEpT9Q3a7IBrx0E6acoYoaHX2Z93UOApPDhgU=
 github.com/Jguer/go-alpm/v2 v2.1.2/go.mod h1:uLQcTMNM904dRiGU+/JDtDdd7Nd8mVbEVaHjhmziT7w=
 github.com/Jguer/votar v1.0.0 h1:drPYpV5Py5BeAQS8xezmT6uCEfLzotNjLf5yfmlHKTg=

local_install.go

@@ -4,7 +4,6 @@ package main
 
 import (
 	"context"
-	"os"
 	"path/filepath"
 	"strings"
 
@@ -35,7 +34,9 @@ func installLocalPKGBUILD(
 		return errors.New(gotext.Get("no target directories specified"))
 	}
 
-	grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, os.Stdout, cmdArgs.ExistsDouble("d", "nodeps"), noCheck)
+	grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm,
+		cmdArgs.ExistsDouble("d", "nodeps"), noCheck, cmdArgs.ExistsArg("needed"),
+		config.Runtime.Logger.Child("grapher"))
 	graph := topo.New[string, *dep.InstallInfo]()
 	for _, target := range cmdArgs.Targets {
 		var errG error

local_install_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"strings"
@@ -18,6 +19,7 @@ import (
 	"github.com/Jguer/yay/v11/pkg/settings"
 	"github.com/Jguer/yay/v11/pkg/settings/exe"
 	"github.com/Jguer/yay/v11/pkg/settings/parser"
+	"github.com/Jguer/yay/v11/pkg/text"
 	"github.com/Jguer/yay/v11/pkg/vcs"
 )
 
@@ -137,6 +139,7 @@ func TestIntegrationLocalInstall(t *testing.T) {
 	config := &settings.Configuration{
 		RemoveMake: "no",
 		Runtime: &settings.Runtime{
+			Logger:     text.NewLogger(io.Discard, strings.NewReader(""), true, "test"),
 			CmdBuilder: cmdBuilder,
 			VCSStore:   &vcs.Mock{},
 			AURCache: &mockaur.MockAUR{
@@ -166,7 +169,7 @@ func TestIntegrationLocalInstall(t *testing.T) {
 }
 
 func TestIntegrationLocalInstallMissingDep(t *testing.T) {
-	wantErr := "could not find dotnet-sdk>=6"
+	wantErr := "could not find dotnet-sdk<7"
 	makepkgBin := t.TempDir() + "/makepkg"
 	pacmanBin := t.TempDir() + "/pacman"
 	gitBin := t.TempDir() + "/git"
@@ -255,6 +258,7 @@ func TestIntegrationLocalInstallMissingDep(t *testing.T) {
 
 	config := &settings.Configuration{
 		Runtime: &settings.Runtime{
+			Logger:     text.NewLogger(io.Discard, strings.NewReader(""), true, "test"),
 			CmdBuilder: cmdBuilder,
 			VCSStore:   &vcs.Mock{},
 			AURCache: &mockaur.MockAUR{
@@ -283,3 +287,161 @@ func TestIntegrationLocalInstallMissingDep(t *testing.T) {
 		assert.Subset(t, strings.Split(show, " "), strings.Split(wantShow[i], " "), fmt.Sprintf("%d - %s", i, show))
 	}
 }
+
+func TestIntegrationLocalInstallNeeded(t *testing.T) {
+	makepkgBin := t.TempDir() + "/makepkg"
+	pacmanBin := t.TempDir() + "/pacman"
+	gitBin := t.TempDir() + "/git"
+	tmpDir := t.TempDir()
+	f, err := os.OpenFile(makepkgBin, os.O_RDONLY|os.O_CREATE, 0o755)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	f, err = os.OpenFile(pacmanBin, os.O_RDONLY|os.O_CREATE, 0o755)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	f, err = os.OpenFile(gitBin, os.O_RDONLY|os.O_CREATE, 0o755)
+	require.NoError(t, err)
+	require.NoError(t, f.Close())
+
+	tars := []string{
+		tmpDir + "/jellyfin-10.8.4-1-x86_64.pkg.tar.zst",
+		tmpDir + "/jellyfin-web-10.8.4-1-x86_64.pkg.tar.zst",
+		tmpDir + "/jellyfin-server-10.8.4-1-x86_64.pkg.tar.zst",
+	}
+
+	wantShow := []string{
+		"makepkg --verifysource -Ccf",
+		"pacman -S --config /etc/pacman.conf -- community/dotnet-sdk-6.0 community/dotnet-runtime-6.0",
+		"pacman -D -q --asdeps --config /etc/pacman.conf -- dotnet-runtime-6.0 dotnet-sdk-6.0",
+		"makepkg --nobuild -fC --ignorearch",
+		"makepkg -c --nobuild --noextract --ignorearch",
+		"makepkg --nobuild -fC --ignorearch",
+		"makepkg -c --nobuild --noextract --ignorearch",
+		"makepkg --nobuild -fC --ignorearch",
+		"makepkg -c --nobuild --noextract --ignorearch",
+	}
+
+	wantCapture := []string{
+		"makepkg --packagelist",
+		"git -C testdata/jfin git reset --hard HEAD",
+		"git -C testdata/jfin git merge --no-edit --ff",
+		"makepkg --packagelist",
+		"makepkg --packagelist",
+	}
+
+	captureOverride := func(cmd *exec.Cmd) (stdout string, stderr string, err error) {
+		return strings.Join(tars, "\n"), "", nil
+	}
+
+	once := sync.Once{}
+
+	showOverride := func(cmd *exec.Cmd) error {
+		once.Do(func() {
+			for _, tar := range tars {
+				f, err := os.OpenFile(tar, os.O_RDONLY|os.O_CREATE, 0o666)
+				require.NoError(t, err)
+				require.NoError(t, f.Close())
+			}
+		})
+		return nil
+	}
+
+	mockRunner := &exe.MockRunner{CaptureFn: captureOverride, ShowFn: showOverride}
+	cmdBuilder := &exe.CmdBuilder{
+		MakepkgBin:       makepkgBin,
+		SudoBin:          "su",
+		PacmanBin:        pacmanBin,
+		PacmanConfigPath: "/etc/pacman.conf",
+		GitBin:           "git",
+		Runner:           mockRunner,
+		SudoLoopEnabled:  false,
+	}
+
+	cmdArgs := parser.MakeArguments()
+	cmdArgs.AddArg("B")
+	cmdArgs.AddArg("i")
+	cmdArgs.AddArg("needed")
+	cmdArgs.AddTarget("testdata/jfin")
+	settings.NoConfirm = true
+	defer func() { settings.NoConfirm = false }()
+	db := &mock.DBExecutor{
+		AlpmArchitecturesFn: func() ([]string, error) {
+			return []string{"x86_64"}, nil
+		},
+		IsCorrectVersionInstalledFn: func(s1, s2 string) bool {
+			return true
+		},
+		LocalPackageFn: func(s string) mock.IPackage {
+			if s == "jellyfin-server" {
+				return &mock.Package{
+					PName:    "jellyfin-server",
+					PBase:    "jellyfin-server",
+					PVersion: "10.8.4-1",
+					PDB:      mock.NewDB("community"),
+				}
+			}
+			return nil
+		},
+		LocalSatisfierExistsFn: func(s string) bool {
+			switch s {
+			case "dotnet-sdk>=6", "dotnet-sdk<7", "dotnet-runtime>=6", "dotnet-runtime<7", "jellyfin-server=10.8.4", "jellyfin-web=10.8.4":
+				return false
+			}
+
+			return true
+		},
+		SyncSatisfierFn: func(s string) mock.IPackage {
+			switch s {
+			case "dotnet-runtime>=6", "dotnet-runtime<7":
+				return &mock.Package{
+					PName:    "dotnet-runtime-6.0",
+					PBase:    "dotnet-runtime-6.0",
+					PVersion: "6.0.100-1",
+					PDB:      mock.NewDB("community"),
+				}
+			case "dotnet-sdk>=6", "dotnet-sdk<7":
+				return &mock.Package{
+					PName:    "dotnet-sdk-6.0",
+					PBase:    "dotnet-sdk-6.0",
+					PVersion: "6.0.100-1",
+					PDB:      mock.NewDB("community"),
+				}
+			}
+
+			return nil
+		},
+	}
+
+	config := &settings.Configuration{
+		RemoveMake: "no",
+		Runtime: &settings.Runtime{
+			Logger:     text.NewLogger(io.Discard, strings.NewReader(""), true, "test"),
+			CmdBuilder: cmdBuilder,
+			VCSStore:   &vcs.Mock{},
+			AURCache: &mockaur.MockAUR{
+				GetFn: func(ctx context.Context, query *aur.Query) ([]aur.Pkg, error) {
+					return []aur.Pkg{}, nil
+				},
+			},
+		},
+	}
+
+	err = handleCmd(context.Background(), config, cmdArgs, db)
+	require.NoError(t, err)
+
+	require.Len(t, mockRunner.ShowCalls, len(wantShow), "show calls: %v", mockRunner.ShowCalls)
+	require.Len(t, mockRunner.CaptureCalls, len(wantCapture))
+
+	for i, call := range mockRunner.ShowCalls {
+		show := call.Args[0].(*exec.Cmd).String()
+		show = strings.ReplaceAll(show, tmpDir, "/testdir") // replace the temp dir with a static path
+		show = strings.ReplaceAll(show, makepkgBin, "makepkg")
+		show = strings.ReplaceAll(show, pacmanBin, "pacman")
+		show = strings.ReplaceAll(show, gitBin, "pacman")
+
+		// options are in a different order on different systems and on CI root user is used
+		assert.Subset(t, strings.Split(show, " "), strings.Split(wantShow[i], " "), fmt.Sprintf("%d - %s", i, show))
+	}
+}

pkg/cmd/graph/main.go

@@ -45,7 +45,9 @@ func handleCmd() error {
 		return errors.Wrap(err, gotext.Get("failed to retrieve aur Cache"))
 	}
 
-	grapher := dep.NewGrapher(dbExecutor, aurCache, true, settings.NoConfirm, os.Stdout, cmdArgs.ExistsDouble("d", "nodeps"), false)
+	grapher := dep.NewGrapher(dbExecutor, aurCache, true, settings.NoConfirm,
+		cmdArgs.ExistsDouble("d", "nodeps"), false, false,
+		config.Runtime.Logger.Child("grapher"))
 
 	return graphPackage(context.Background(), grapher, cmdArgs.Targets)
 }
@@ -66,7 +68,7 @@ func graphPackage(
 		return errors.New(gotext.Get("only one target is allowed"))
 	}
 
-	graph, err := grapher.GraphFromAURCache(ctx, nil, []string{targets[0]})
+	graph, err := grapher.GraphFromAUR(ctx, nil, []string{targets[0]})
 	if err != nil {
 		return err
 	}

pkg/db/mock/executor.go

@@ -16,6 +16,7 @@ type (
 
 type DBExecutor struct {
 	db.Executor
+	LocalPackageFn                func(string) IPackage
 	IsCorrectVersionInstalledFn   func(string, string) bool
 	SyncPackageFn                 func(string) IPackage
 	PackagesFromGroupFn           func(string) []IPackage
@@ -69,7 +70,11 @@ func (t *DBExecutor) LastBuildTime() time.Time {
 }
 
 func (t *DBExecutor) LocalPackage(s string) IPackage {
-	return nil
+	if t.LocalPackageFn != nil {
+		return t.LocalPackageFn(s)
+	}
+
+	panic("implement me")
 }
 
 func (t *DBExecutor) LocalPackages() []IPackage {

pkg/dep/dep_graph.go

@@ -3,20 +3,19 @@ package dep
 import (
 	"context"
 	"fmt"
-	"io"
-	"os"
 	"strconv"
 
+	aurc "github.com/Jguer/aur"
+	alpm "github.com/Jguer/go-alpm/v2"
+	gosrc "github.com/Morganamilo/go-srcinfo"
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/leonelquinteros/gotext"
+
 	"github.com/Jguer/yay/v11/pkg/db"
 	"github.com/Jguer/yay/v11/pkg/intrange"
 	aur "github.com/Jguer/yay/v11/pkg/query"
 	"github.com/Jguer/yay/v11/pkg/text"
 	"github.com/Jguer/yay/v11/pkg/topo"
-
-	aurc "github.com/Jguer/aur"
-	alpm "github.com/Jguer/go-alpm/v2"
-	gosrc "github.com/Morganamilo/go-srcinfo"
-	"github.com/leonelquinteros/gotext"
 )
 
 type InstallInfo struct {
@@ -94,29 +93,32 @@ var colorMap = map[Reason]string{
 }
 
 type Grapher struct {
-	dbExecutor  db.Executor
+	logger        *text.Logger
-	aurCache    aurc.QueryClient
+	providerCache map[string][]aur.Pkg
-	fullGraph   bool // If true, the graph will include all dependencies including already installed ones or repo
-	noConfirm   bool
-	noDeps      bool      // If true, the graph will not include dependencies
-	noCheckDeps bool      // If true, the graph will not include dependencies
-	w           io.Writer // output writer
 
-	providerCache map[string]*aur.Pkg
+	dbExecutor  db.Executor
+	aurClient   aurc.QueryClient
+	fullGraph   bool // If true, the graph will include all dependencies including already installed ones or repo
+	noConfirm   bool // If true, the graph will not prompt for confirmation
+	noDeps      bool // If true, the graph will not include dependencies
+	noCheckDeps bool // If true, the graph will not include check dependencies
+	needed      bool // If true, the graph will only include packages that are not installed
 }
 
 func NewGrapher(dbExecutor db.Executor, aurCache aurc.QueryClient,
-	fullGraph, noConfirm bool, output io.Writer, noDeps bool, noCheckDeps bool,
+	fullGraph, noConfirm, noDeps, noCheckDeps, needed bool,
+	logger *text.Logger,
 ) *Grapher {
 	return &Grapher{
 		dbExecutor:    dbExecutor,
-		aurCache:      aurCache,
+		aurClient:     aurCache,
 		fullGraph:     fullGraph,
 		noConfirm:     noConfirm,
-		w:             output,
 		noDeps:        noDeps,
 		noCheckDeps:   noCheckDeps,
-		providerCache: make(map[string]*aurc.Pkg, 5),
+		needed:        needed,
+		providerCache: make(map[string][]aurc.Pkg, 5),
+		logger:        logger,
 	}
 }
 
@@ -127,11 +129,10 @@ func (g *Grapher) GraphFromTargets(ctx context.Context,
 		graph = topo.New[string, *InstallInfo]()
 	}
 
+	aurTargets := make([]string, 0, len(targets))
+
 	for _, targetString := range targets {
-		var (
+		target := ToTarget(targetString)
-			err    error
-			target = ToTarget(targetString)
-		)
 
 		switch target.DB {
 		case "": // unspecified db
@@ -179,7 +180,7 @@ func (g *Grapher) GraphFromTargets(ctx context.Context,
 
 			fallthrough
 		case "aur":
-			graph, err = g.GraphFromAURCache(ctx, graph, []string{target.Name})
+			aurTargets = append(aurTargets, target.Name)
 		default:
 			graph.AddNode(target.Name)
 			g.ValidateAndSetNodeInfo(graph, target.Name, &topo.NodeInfo[*InstallInfo]{
@@ -193,10 +194,12 @@ func (g *Grapher) GraphFromTargets(ctx context.Context,
 				},
 			})
 		}
+	}
 
-		if err != nil {
+	var errA error
-			return nil, err
+	graph, errA = g.GraphFromAUR(ctx, graph, aurTargets)
-		}
+	if errA != nil {
+		return nil, errA
 	}
 
 	return graph, nil
@@ -205,13 +208,13 @@ func (g *Grapher) GraphFromTargets(ctx context.Context,
 func (g *Grapher) pickSrcInfoPkgs(pkgs []aurc.Pkg) ([]aurc.Pkg, error) {
 	final := make([]aurc.Pkg, 0, len(pkgs))
 	for i := range pkgs {
-		fmt.Fprintln(os.Stdout, text.Magenta(strconv.Itoa(i+1)+" ")+text.Bold(pkgs[i].Name)+
+		g.logger.Println(text.Magenta(strconv.Itoa(i+1)+" ") + text.Bold(pkgs[i].Name) +
-			" "+text.Cyan(pkgs[i].Version))
+			" " + text.Cyan(pkgs[i].Version))
-		fmt.Fprintln(os.Stdout, "    "+pkgs[i].Description)
+		g.logger.Println("    " + pkgs[i].Description)
 	}
-	text.Infoln(gotext.Get("Packages to exclude") + " (eg: \"1 2 3\", \"1-3\", \"^4\"):")
+	g.logger.Infoln(gotext.Get("Packages to exclude") + " (eg: \"1 2 3\", \"1-3\", \"^4\"):")
 
-	numberBuf, err := text.GetInput(os.Stdin, "", g.noConfirm)
+	numberBuf, err := g.logger.GetInput("", g.noConfirm)
 	if err != nil {
 		return nil, err
 	}
@@ -328,7 +331,7 @@ func (g *Grapher) GraphAURTarget(ctx context.Context,
 	return graph
 }
 
-func (g *Grapher) GraphFromAURCache(ctx context.Context,
+func (g *Grapher) GraphFromAUR(ctx context.Context,
 	graph *topo.Graph[string, *InstallInfo],
 	targets []string,
 ) (*topo.Graph[string, *InstallInfo], error) {
@@ -336,27 +339,150 @@ func (g *Grapher) GraphFromAURCache(ctx context.Context,
 		graph = topo.New[string, *InstallInfo]()
 	}
 
+	if len(targets) == 0 {
+		return graph, nil
+	}
+
+	aurPkgs, errCache := g.aurClient.Get(ctx, &aurc.Query{By: aurc.Name, Needles: targets})
+	if errCache != nil {
+		text.Errorln(errCache)
+	}
+
+	for i := range aurPkgs {
+		pkg := &aurPkgs[i]
+		if _, ok := g.providerCache[pkg.Name]; !ok {
+			g.providerCache[pkg.Name] = []aurc.Pkg{*pkg}
+		}
+	}
+
 	for _, target := range targets {
-		aurPkgs, _ := g.aurCache.Get(ctx, &aurc.Query{By: aurc.Name, Needles: []string{target}})
+		if cachedProvidePkg, ok := g.providerCache[target]; ok {
+			aurPkgs = cachedProvidePkg
+		} else {
+			var errA error
+			aurPkgs, errA = g.aurClient.Get(ctx, &aurc.Query{By: aurc.Provides, Needles: []string{target}, Contains: true})
+			if errA != nil {
+				g.logger.Errorln(gotext.Get("Failed to find AUR package for"), target, ":", errA)
+			}
+		}
+
 		if len(aurPkgs) == 0 {
-			text.Errorln("No AUR package found for", target)
+			g.logger.Errorln(gotext.Get("No AUR package found for"), " ", target)
 
 			continue
 		}
 
-		pkg := provideMenu(g.w, target, aurPkgs, g.noConfirm)
+		aurPkg := &aurPkgs[0]
+		if len(aurPkgs) > 1 {
+			chosen := g.provideMenu(target, aurPkgs)
+			aurPkg = chosen
+			g.providerCache[target] = []aurc.Pkg{*aurPkg}
+		}
 
-		graph = g.GraphAURTarget(ctx, graph, pkg, &InstallInfo{
+		if g.needed {
-			AURBase: &pkg.PackageBase,
+			if pkg := g.dbExecutor.LocalPackage(aurPkg.Name); pkg != nil {
+				if db.VerCmp(pkg.Version(), aurPkg.Version) >= 0 {
+					g.logger.Warnln(gotext.Get("%s is up to date -- skipping", text.Cyan(pkg.Name()+"-"+pkg.Version())))
+					continue
+				}
+			}
+		}
+
+		graph = g.GraphAURTarget(ctx, graph, aurPkg, &InstallInfo{
+			AURBase: &aurPkg.PackageBase,
 			Reason:  Explicit,
 			Source:  AUR,
-			Version: pkg.Version,
+			Version: aurPkg.Version,
 		})
 	}
 
 	return graph, nil
 }
 
+// Removes found deps from the deps mapset and returns the found deps.
+func (g *Grapher) findDepsFromAUR(ctx context.Context,
+	deps mapset.Set[string],
+) []aurc.Pkg {
+	pkgsToAdd := make([]aurc.Pkg, 0, deps.Cardinality())
+	if deps.Cardinality() == 0 {
+		return []aurc.Pkg{}
+	}
+
+	missingNeedles := make([]string, 0, deps.Cardinality())
+	for _, depString := range deps.ToSlice() {
+		if _, ok := g.providerCache[depString]; !ok {
+			depName, _, _ := splitDep(depString)
+			missingNeedles = append(missingNeedles, depName)
+		}
+	}
+
+	if len(missingNeedles) != 0 {
+		g.logger.Debugln("deps to find", missingNeedles)
+		// provider search is more demanding than a simple search
+		// try to find name match if possible and then try to find provides.
+		aurPkgs, errCache := g.aurClient.Get(ctx, &aurc.Query{
+			By: aurc.Name, Needles: missingNeedles, Contains: false,
+		})
+		if errCache != nil {
+			text.Errorln(errCache)
+		}
+
+		for i := range aurPkgs {
+			pkg := &aurPkgs[i]
+			for _, val := range pkg.Provides {
+				if deps.Contains(val) {
+					g.providerCache[val] = append(g.providerCache[val], *pkg)
+				}
+			}
+
+			if deps.Contains(pkg.Name) {
+				g.providerCache[pkg.Name] = append(g.providerCache[pkg.Name], *pkg)
+			}
+		}
+	}
+
+	for _, depString := range deps.ToSlice() {
+		var aurPkgs []aurc.Pkg
+		depName, _, _ := splitDep(depString)
+
+		if cachedProvidePkg, ok := g.providerCache[depString]; ok {
+			aurPkgs = cachedProvidePkg
+		} else {
+			var errA error
+			aurPkgs, errA = g.aurClient.Get(ctx, &aurc.Query{By: aurc.Provides, Needles: []string{depName}, Contains: true})
+			if errA != nil {
+				g.logger.Errorln(gotext.Get("Failed to find AUR package for"), depString, ":", errA)
+			}
+		}
+
+		// remove packages that don't satisfy the dependency
+		for i := 0; i < len(aurPkgs); i++ {
+			if !satisfiesAur(depString, &aurPkgs[i]) {
+				aurPkgs = append(aurPkgs[:i], aurPkgs[i+1:]...)
+				i--
+			}
+		}
+
+		if len(aurPkgs) == 0 {
+			g.logger.Errorln(gotext.Get("No AUR package found for"), " ", depString)
+
+			continue
+		}
+
+		pkg := aurPkgs[0]
+		if len(aurPkgs) > 1 {
+			chosen := g.provideMenu(depString, aurPkgs)
+			pkg = *chosen
+		}
+
+		g.providerCache[depString] = []aurc.Pkg{pkg}
+		deps.Remove(depString)
+		pkgsToAdd = append(pkgsToAdd, pkg)
+	}
+
+	return pkgsToAdd
+}
+
 func (g *Grapher) ValidateAndSetNodeInfo(graph *topo.Graph[string, *InstallInfo],
 	node string, nodeInfo *topo.NodeInfo[*InstallInfo],
 ) {
@@ -377,110 +503,106 @@ func (g *Grapher) addNodes(
 	deps []string,
 	depType Reason,
 ) {
-	for _, depString := range deps {
+	targetsToFind := mapset.NewThreadUnsafeSet(deps...)
-		depName, mod, ver := splitDep(depString)
+	// Check if in graph already
-
+	for _, depString := range targetsToFind.ToSlice() {
-		if g.dbExecutor.LocalSatisfierExists(depString) {
+		if !graph.Exists(depString) {
-			if g.fullGraph {
-				g.ValidateAndSetNodeInfo(
-					graph,
-					depName,
-					&topo.NodeInfo[*InstallInfo]{Color: colorMap[depType], Background: bgColorMap[Local]})
-
-				if err := graph.DependOn(depName, parentPkgName); err != nil {
-					text.Warnln(depName, parentPkgName, err)
-				}
-			}
-
 			continue
 		}
 
-		if graph.Exists(depName) {
+		if err := graph.DependOn(depString, parentPkgName); err != nil {
-			if err := graph.DependOn(depName, parentPkgName); err != nil {
+			g.logger.Warnln(depString, parentPkgName, err)
-				text.Warnln(depName, parentPkgName, err)
+		}
-			}
 
+		targetsToFind.Remove(depString)
+	}
+
+	// Check installed
+	for _, depString := range targetsToFind.ToSlice() {
+		depName, _, _ := splitDep(depString)
+		if !g.dbExecutor.LocalSatisfierExists(depString) {
 			continue
 		}
 
-		// Check ALPM
+		if g.fullGraph {
-		if alpmPkg := g.dbExecutor.SyncSatisfier(depString); alpmPkg != nil {
-			if err := graph.DependOn(alpmPkg.Name(), parentPkgName); err != nil {
-				text.Warnln("repo dep warn:", depName, parentPkgName, err)
-			}
-
-			dbName := alpmPkg.DB().Name()
 			g.ValidateAndSetNodeInfo(
 				graph,
-				alpmPkg.Name(),
+				depName,
-				&topo.NodeInfo[*InstallInfo]{
+				&topo.NodeInfo[*InstallInfo]{Color: colorMap[depType], Background: bgColorMap[Local]})
-					Color:      colorMap[depType],
-					Background: bgColorMap[Sync],
-					Value: &InstallInfo{
-						Source:     Sync,
-						Reason:     depType,
-						Version:    alpmPkg.Version(),
-						SyncDBName: &dbName,
-					},
-				})
 
-			if newDeps := alpmPkg.Depends().Slice(); len(newDeps) != 0 && g.fullGraph {
+			if err := graph.DependOn(depName, parentPkgName); err != nil {
-				newDepsSlice := make([]string, 0, len(newDeps))
+				g.logger.Warnln(depName, parentPkgName, err)
-				for _, newDep := range newDeps {
-					newDepsSlice = append(newDepsSlice, newDep.Name)
-				}
-
-				g.addNodes(ctx, graph, alpmPkg.Name(), newDepsSlice, Dep)
 			}
+		}
 
+		targetsToFind.Remove(depString)
+	}
+
+	// Check Sync
+	for _, depString := range targetsToFind.ToSlice() {
+		alpmPkg := g.dbExecutor.SyncSatisfier(depString)
+		if alpmPkg == nil {
 			continue
 		}
 
-		var aurPkgs []aur.Pkg
+		if err := graph.DependOn(alpmPkg.Name(), parentPkgName); err != nil {
-		if cachedProvidePkg, ok := g.providerCache[depName]; ok {
+			g.logger.Warnln("repo dep warn:", depString, parentPkgName, err)
-			aurPkgs = []aur.Pkg{*cachedProvidePkg}
-		} else {
-			var errMeta error
-			aurPkgs, errMeta = g.aurCache.Get(ctx,
-				&aurc.Query{
-					Needles:  []string{depName},
-					By:       aurc.None,
-					Contains: false,
-				})
-			if errMeta != nil {
-				text.Warnln("AUR cache error:", errMeta)
-			}
 		}
 
-		if len(aurPkgs) != 0 { // Check AUR
+		dbName := alpmPkg.DB().Name()
-			pkg := aurPkgs[0]
+		g.ValidateAndSetNodeInfo(
-			if len(aurPkgs) > 1 {
+			graph,
-				chosen := provideMenu(g.w, depName, aurPkgs, g.noConfirm)
+			alpmPkg.Name(),
-				g.providerCache[depName] = chosen
+			&topo.NodeInfo[*InstallInfo]{
-				pkg = *chosen
+				Color:      colorMap[depType],
+				Background: bgColorMap[Sync],
+				Value: &InstallInfo{
+					Source:     Sync,
+					Reason:     depType,
+					Version:    alpmPkg.Version(),
+					SyncDBName: &dbName,
+				},
+			})
+
+		if newDeps := alpmPkg.Depends().Slice(); len(newDeps) != 0 && g.fullGraph {
+			newDepsSlice := make([]string, 0, len(newDeps))
+			for _, newDep := range newDeps {
+				newDepsSlice = append(newDepsSlice, newDep.Name)
 			}
 
-			if err := graph.DependOn(pkg.Name, parentPkgName); err != nil {
+			g.addNodes(ctx, graph, alpmPkg.Name(), newDepsSlice, Dep)
-				text.Warnln("aur dep warn:", pkg.Name, parentPkgName, err)
-			}
-
-			graph.SetNodeInfo(
-				pkg.Name,
-				&topo.NodeInfo[*InstallInfo]{
-					Color:      colorMap[depType],
-					Background: bgColorMap[AUR],
-					Value: &InstallInfo{
-						Source:  AUR,
-						Reason:  depType,
-						AURBase: &pkg.PackageBase,
-						Version: pkg.Version,
-					},
-				})
-			g.addDepNodes(ctx, &pkg, graph)
-
-			continue
 		}
 
+		targetsToFind.Remove(depString)
+	}
+
+	// Check AUR
+	pkgsToAdd := g.findDepsFromAUR(ctx, targetsToFind)
+	for i := range pkgsToAdd {
+		aurPkg := &pkgsToAdd[i]
+		if err := graph.DependOn(aurPkg.Name, parentPkgName); err != nil {
+			g.logger.Warnln("aur dep warn:", aurPkg.Name, parentPkgName, err)
+		}
+
+		graph.SetNodeInfo(
+			aurPkg.Name,
+			&topo.NodeInfo[*InstallInfo]{
+				Color:      colorMap[depType],
+				Background: bgColorMap[AUR],
+				Value: &InstallInfo{
+					Source:  AUR,
+					Reason:  depType,
+					AURBase: &aurPkg.PackageBase,
+					Version: aurPkg.Version,
+				},
+			})
+
+		g.addDepNodes(ctx, aurPkg, graph)
+	}
+
+	// Add missing to graph
+	for _, depString := range targetsToFind.ToSlice() {
+		depName, mod, ver := splitDep(depString)
 		// no dep found. add as missing
 		graph.AddNode(depName)
 		graph.SetNodeInfo(depName, &topo.NodeInfo[*InstallInfo]{
@@ -495,7 +617,7 @@ func (g *Grapher) addNodes(
 	}
 }
 
-func provideMenu(w io.Writer, dep string, options []aur.Pkg, noConfirm bool) *aur.Pkg {
+func (g *Grapher) provideMenu(dep string, options []aur.Pkg) *aur.Pkg {
 	size := len(options)
 	if size == 1 {
 		return &options[0]
@@ -505,27 +627,27 @@ func provideMenu(w io.Writer, dep string, options []aur.Pkg, noConfirm bool) *au
 	str += "\n"
 
 	size = 1
-	str += text.SprintOperationInfo(gotext.Get("Repository AUR"), "\n    ")
+	str += g.logger.SprintOperationInfo(gotext.Get("Repository AUR"), "\n    ")
 
 	for i := range options {
 		str += fmt.Sprintf("%d) %s ", size, options[i].Name)
 		size++
 	}
 
-	text.OperationInfoln(str)
+	g.logger.OperationInfoln(str)
 
 	for {
-		fmt.Fprintln(w, gotext.Get("\nEnter a number (default=1): "))
+		g.logger.Println(gotext.Get("\nEnter a number (default=1): "))
 
-		if noConfirm {
+		if g.noConfirm {
-			fmt.Fprintln(w, "1")
+			g.logger.Println("1")
 
 			return &options[0]
 		}
 
-		numberBuf, err := text.GetInput(os.Stdin, "", false)
+		numberBuf, err := g.logger.GetInput("", false)
 		if err != nil {
-			fmt.Fprintln(os.Stderr, err)
+			g.logger.Errorln(err)
 
 			break
 		}
@@ -536,13 +658,14 @@ func provideMenu(w io.Writer, dep string, options []aur.Pkg, noConfirm bool) *au
 
 		num, err := strconv.Atoi(numberBuf)
 		if err != nil {
-			text.Errorln(gotext.Get("invalid number: %s", numberBuf))
+			g.logger.Errorln(gotext.Get("invalid number: %s", numberBuf))
 
 			continue
 		}
 
 		if num < 1 || num >= size {
-			text.Errorln(gotext.Get("invalid value: %d is not between %d and %d", num, 1, size-1))
+			g.logger.Errorln(gotext.Get("invalid value: %d is not between %d and %d",
+				num, 1, size-1))
 
 			continue
 		}

pkg/dep/dep_graph_test.go

@@ -3,6 +3,7 @@ package dep
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"testing"
@@ -14,6 +15,7 @@ import (
 	"github.com/Jguer/yay/v11/pkg/db/mock"
 	mockaur "github.com/Jguer/yay/v11/pkg/dep/mock"
 	aur "github.com/Jguer/yay/v11/pkg/query"
+	"github.com/Jguer/yay/v11/pkg/text"
 )
 
 func ptrString(s string) *string {
@@ -88,7 +90,7 @@ func TestGrapher_GraphFromTargets_jellyfin(t *testing.T) {
 			return jfinServerFn(ctx, query)
 		}
 
-		panic("implement me")
+		panic(fmt.Sprintf("implement me %v", query.Needles))
 	}}
 
 	type fields struct {
@@ -193,8 +195,9 @@ func TestGrapher_GraphFromTargets_jellyfin(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			g := NewGrapher(tt.fields.dbExecutor,
-				tt.fields.aurCache, false, true, os.Stdout,
+				tt.fields.aurCache, false, true,
-				tt.fields.noDeps, tt.fields.noCheckDeps)
+				tt.fields.noDeps, tt.fields.noCheckDeps, false,
+				text.NewLogger(io.Discard, &os.File{}, true, "test"))
 			got, err := g.GraphFromTargets(context.Background(), nil, tt.args.targets)
 			require.NoError(t, err)
 			layers := got.TopoSortedLayerMap(nil)

pkg/query/types_test.go

@@ -134,7 +134,7 @@ func Test_aurQuery_printSearch(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			w := &strings.Builder{}
-			executor := &mock.DBExecutor{}
+			executor := &mock.DBExecutor{LocalPackageFn: func(string) mock.IPackage { return nil }}
 			text.UseColor = tt.useColor
 
 			// Fire
@@ -236,7 +236,7 @@ func Test_repoQuery_printSearch(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			w := &strings.Builder{}
-			executor := &mock.DBExecutor{}
+			executor := &mock.DBExecutor{LocalPackageFn: func(string) mock.IPackage { return nil }}
 			text.UseColor = tt.useColor
 
 			// Fire

pkg/settings/config.go

@@ -235,10 +235,10 @@ func DefaultConfig(version string) *Configuration {
 		UseAsk:                 false,
 		CombinedUpgrade:        false,
 		SeparateSources:        true,
-		NewInstallEngine:       false,
+		NewInstallEngine:       true,
 		Version:                version,
 		Debug:                  false,
-		UseRPC:                 false,
+		UseRPC:                 true,
 	}
 }
 

pkg/upgrade/service_test.go

@@ -250,7 +250,8 @@ func TestUpgradeService_GraphUpgrades(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			grapher := dep.NewGrapher(dbExe, mockAUR,
-				false, true, io.Discard, false, false)
+				false, true, false, false, false, text.NewLogger(tt.fields.output,
+					tt.fields.input, true, "test"))
 
 			cfg := &settings.Configuration{
 				Runtime: &settings.Runtime{Mode: parser.ModeAny},
@@ -258,7 +259,8 @@ func TestUpgradeService_GraphUpgrades(t *testing.T) {
 			}
 
 			u := &UpgradeService{
-				log:        text.NewLogger(tt.fields.output, tt.fields.input, true, "test"),
+				log: text.NewLogger(tt.fields.output,
+					tt.fields.input, true, "test"),
 				grapher:    grapher,
 				aurCache:   mockAUR,
 				dbExecutor: dbExe,

sync.go

@@ -45,7 +45,8 @@ func syncInstall(ctx context.Context,
 		}
 	}
 
-	grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm, os.Stdout, noDeps, noCheck)
+	grapher := dep.NewGrapher(dbExecutor, aurCache, false, settings.NoConfirm,
+		noDeps, noCheck, cmdArgs.ExistsArg("needed"), config.Runtime.Logger.Child("grapher"))
 
 	graph, err := grapher.GraphFromTargets(ctx, nil, cmdArgs.Targets)
 	if err != nil {