feat(download): download PKGBUILD repos interface

.golangci.yml

@@ -54,7 +54,6 @@ linters:
     - gocritic
     - gofmt
     - goimports
-    - golint
     - goprintffuncname
     - gosec
     - gosimple
@@ -63,6 +62,8 @@ linters:
     - lll
     - misspell
     - nakedret
+    - prealloc
+    - revive
     - rowserrcheck
     - staticcheck
     - structcheck
@@ -73,7 +74,6 @@ linters:
     - unused
     - varcheck
     - whitespace
-    - prealloc
 
     # disabled want to fix
     #- scopelint

cmd.go

@@ -277,7 +277,9 @@ func handleGetpkgbuild(cmdArgs *settings.Arguments, dbExecutor db.Executor) erro
 	if cmdArgs.ExistsArg("p", "print") {
 		return printPkgbuilds(dbExecutor, config.Runtime.HTTPClient, cmdArgs.Targets)
 	}
-	return getPkgbuilds(cmdArgs.Targets, dbExecutor, cmdArgs.ExistsArg("f", "force"))
+	return getPkgbuilds(dbExecutor,
+		config.Runtime.CmdRunner, config.Runtime.CmdBuilder,
+		cmdArgs.Targets, config.Runtime.Mode, config.AURURL, cmdArgs.ExistsArg("f", "force"))
 }
 
 func handleYogurt(cmdArgs *settings.Arguments, dbExecutor db.Executor) error {

download.go

@@ -3,20 +3,15 @@ package main
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/leonelquinteros/gotext"
-	"github.com/pkg/errors"
-
-	alpm "github.com/Jguer/go-alpm/v2"
 
 	"github.com/Jguer/yay/v10/pkg/db"
-	"github.com/Jguer/yay/v10/pkg/dep"
+	"github.com/Jguer/yay/v10/pkg/download"
-	"github.com/Jguer/yay/v10/pkg/multierror"
+	"github.com/Jguer/yay/v10/pkg/settings"
-	"github.com/Jguer/yay/v10/pkg/query"
+	"github.com/Jguer/yay/v10/pkg/settings/exe"
 	"github.com/Jguer/yay/v10/pkg/text"
 )
 
@@ -80,34 +75,6 @@ func gitHasDiff(path, name string) (bool, error) {
 	return true, nil
 }
 
-// TODO: yay-next passes args through the header, use that to unify ABS and AUR
-func gitDownloadABS(url, path, name string) (bool, error) {
-	if err := os.MkdirAll(path, 0o755); err != nil {
-		return false, err
-	}
-
-	if _, errExist := os.Stat(filepath.Join(path, name)); os.IsNotExist(errExist) {
-		cmd := config.Runtime.CmdBuilder.BuildGitCmd(path, "clone", "--no-progress", "--single-branch",
-			"-b", "packages/"+name, url, name)
-		_, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0)
-		if err != nil {
-			return false, fmt.Errorf(gotext.Get("error cloning %s: %s", name, stderr))
-		}
-
-		return true, nil
-	} else if errExist != nil {
-		return false, fmt.Errorf(gotext.Get("error reading %s", filepath.Join(path, name, ".git")))
-	}
-
-	cmd := config.Runtime.CmdBuilder.BuildGitCmd(filepath.Join(path, name), "pull", "--ff-only")
-	_, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0)
-	if err != nil {
-		return false, fmt.Errorf(gotext.Get("error fetching %s: %s", name, stderr))
-	}
-
-	return true, nil
-}
-
 func gitDownload(url, path, name string) (bool, error) {
 	_, err := os.Stat(filepath.Join(path, name, ".git"))
 	if os.IsNotExist(err) {
@@ -149,173 +116,33 @@ func gitMerge(path, name string) error {
 	return nil
 }
 
-func getPkgbuilds(pkgs []string, dbExecutor db.Executor, force bool) error {
+func getPkgbuilds(dbExecutor db.Executor,
-	missing := false
+	cmdRunner exe.Runner,
+	cmdBuilder exe.GitCmdBuilder, targets []string,
+	mode settings.TargetMode,
+	aurURL string,
+	force bool) error {
 	wd, err := os.Getwd()
 	if err != nil {
 		return err
 	}
 
-	pkgs = query.RemoveInvalidTargets(pkgs, config.Runtime.Mode)
+	cloned, errD := download.PKGBUILDRepos(dbExecutor, cmdRunner, cmdBuilder, targets, mode, aurURL, wd, force)
-	aur, repo := packageSlices(pkgs, dbExecutor)
+	if errD != nil {
-
+		text.Errorln(errD)
-	for n := range aur {
-		_, pkg := text.SplitDBFromName(aur[n])
-		aur[n] = pkg
 	}
 
-	info, err := query.AURInfoPrint(config.Runtime.AURClient, aur, config.RequestSplitN)
+	if len(targets) != len(cloned) {
-	if err != nil {
+		missing := []string{}
-		return err
+		for _, target := range targets {
+			if _, ok := cloned[target]; !ok {
+				missing = append(missing, target)
 			}
+		}
+		text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", "))
 
-	if len(repo) > 0 {
-		missing, err = getPkgbuildsfromABS(repo, wd, dbExecutor, force)
-		if err != nil {
-			return err
-		}
-	}
-
-	if len(aur) > 0 {
-		allBases := dep.GetBases(info)
-		bases := make([]dep.Base, 0)
-
-		for _, base := range allBases {
-			name := base.Pkgbase()
-			pkgDest := filepath.Join(wd, name)
-			_, err = os.Stat(pkgDest)
-			if os.IsNotExist(err) {
-				bases = append(bases, base)
-			} else if err != nil {
-				text.Errorln(err)
-				continue
-			} else {
-				if force {
-					if err = os.RemoveAll(pkgDest); err != nil {
-						text.Errorln(err)
-						continue
-					}
-					bases = append(bases, base)
-				} else {
-					text.Warnln(gotext.Get("%s already exists. Use -f/--force to overwrite", pkgDest))
-					continue
-				}
-			}
-		}
-
-		if _, err = downloadPkgbuilds(bases, nil, wd); err != nil {
-			return err
-		}
-
-		missing = missing || len(aur) != len(info)
-	}
-
-	if missing {
 		err = fmt.Errorf("")
 	}
 
 	return err
 }
-
-// GetPkgbuild downloads pkgbuild from the ABS.
-func getPkgbuildsfromABS(pkgs []string, path string, dbExecutor db.Executor, force bool) (bool, error) {
-	var wg sync.WaitGroup
-	var mux sync.Mutex
-	var errs multierror.MultiError
-	names := make(map[string]string)
-	missing := make([]string, 0)
-	downloaded := 0
-
-	for _, pkgN := range pkgs {
-		var pkg alpm.IPackage
-		var err error
-		var url string
-		pkgDB, name := text.SplitDBFromName(pkgN)
-
-		if pkgDB != "" {
-			pkg = dbExecutor.SatisfierFromDB(name, pkgDB)
-		} else {
-			pkg = dbExecutor.SyncSatisfier(name)
-		}
-
-		if pkg == nil {
-			missing = append(missing, name)
-			continue
-		}
-
-		name = pkg.Base()
-		if name == "" {
-			name = pkg.Name()
-		}
-
-		// TODO: Check existence with ls-remote
-		// https://git.archlinux.org/svntogit/packages.git
-		switch pkg.DB().Name() {
-		case "core", "extra", "testing":
-			url = "https://github.com/archlinux/svntogit-packages.git"
-		case "community", "multilib", "community-testing", "multilib-testing":
-			url = "https://github.com/archlinux/svntogit-community.git"
-		default:
-			missing = append(missing, name)
-			continue
-		}
-
-		_, err = os.Stat(filepath.Join(path, name))
-		switch {
-		case err != nil && !os.IsNotExist(err):
-			text.Errorln(err)
-			continue
-		case os.IsNotExist(err), force:
-			if err = os.RemoveAll(filepath.Join(path, name)); err != nil {
-				text.Errorln(err)
-				continue
-			}
-		default:
-			text.Warn(gotext.Get("%s already downloaded -- use -f to overwrite", text.Cyan(name)))
-			continue
-		}
-
-		names[name] = url
-	}
-
-	if len(missing) != 0 {
-		text.Warnln(gotext.Get("Missing ABS packages:"),
-			text.Cyan(strings.Join(missing, ", ")))
-	}
-
-	download := func(pkg string, url string) {
-		defer wg.Done()
-		if _, err := gitDownloadABS(url, config.ABSDir, pkg); err != nil {
-			errs.Add(errors.New(gotext.Get("failed to get pkgbuild: %s: %s", text.Cyan(pkg), err.Error())))
-			return
-		}
-
-		_, stderr, err := config.Runtime.CmdRunner.Capture(
-			exec.Command(
-				"cp", "-r",
-				filepath.Join(config.ABSDir, pkg, "trunk"),
-				filepath.Join(path, pkg)), 0)
-		mux.Lock()
-		downloaded++
-		if err != nil {
-			errs.Add(errors.New(gotext.Get("failed to link %s: %s", text.Cyan(pkg), stderr)))
-		} else {
-			fmt.Fprintln(os.Stdout, gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s", downloaded, len(names), text.Cyan(pkg)))
-		}
-		mux.Unlock()
-	}
-
-	count := 0
-	for name, url := range names {
-		wg.Add(1)
-		go download(name, url)
-		count++
-		if count%25 == 0 {
-			wg.Wait()
-		}
-	}
-
-	wg.Wait()
-
-	return len(missing) != 0, errs.Return()
-}

pkg/download/abs.go

@@ -7,6 +7,8 @@ import (
 	"net/http"
 
 	"github.com/leonelquinteros/gotext"
+
+	"github.com/Jguer/yay/v10/pkg/settings/exe"
 )
 
 const (
@@ -21,22 +23,37 @@ var (
 	ABSCommunityURL       = "https://github.com/archlinux/svntogit-community"
 )
 
-// Return format for pkgbuild
+func getRepoURL(db string) (string, error) {
-// https://github.com/archlinux/svntogit-community/raw/packages/neovim/trunk/PKGBUILD
-func getPackageURL(db, pkgName string) (string, error) {
-	repoURL := ""
 	switch db {
 	case "core", "extra", "testing":
-		repoURL = ABSPackageURL
+		return ABSPackageURL, nil
 	case "community", "multilib", "community-testing", "multilib-testing":
-		repoURL = ABSCommunityURL
+		return ABSCommunityURL, nil
-	default:
+	}
+
 	return "", ErrInvalidRepository
 }
 
-	return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), nil
+// Return format for pkgbuild
+// https://github.com/archlinux/svntogit-community/raw/packages/neovim/trunk/PKGBUILD
+func getPackageURL(db, pkgName string) (string, error) {
+	repoURL, err := getRepoURL(db)
+	if err != nil {
+		return "", err
 	}
 
+	return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), err
+}
+
+// Return format for pkgbuild repo
+// https://github.com/archlinux/svntogit-community.git
+func getPackageRepoURL(db string) (string, error) {
+	repoURL, err := getRepoURL(db)
+
+	return repoURL + ".git", err
+}
+
+// GetABSPkgbuild retrieves the PKGBUILD file to a dest directory
 func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, error) {
 	packageURL, err := getPackageURL(dbName, pkgName)
 	if err != nil {
@@ -48,7 +65,7 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er
 		return nil, err
 	}
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		return nil, ErrABSPackageNotFound
 	}
 
@@ -61,3 +78,14 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er
 
 	return pkgBuild, nil
 }
+
+// ABSPkgbuildRepo retrieves the PKGBUILD repository to a dest directory
+func ABSPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, dbName, pkgName, dest string, force bool) error {
+	pkgURL, err := getPackageRepoURL(dbName)
+	if err != nil {
+		return err
+	}
+
+	return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL,
+		pkgName, dest, force, "--single-branch", "-b", "packages/"+pkgName)
+}

pkg/download/abs_test.go

@@ -81,13 +81,11 @@ func Test_getPackageURL(t *testing.T) {
 
 func TestGetABSPkgbuild(t *testing.T) {
 	pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
 		w.WriteHeader(200)
 		w.Write([]byte(gitExtrasPKGBUILD))
 	})
 
 	notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
 		w.WriteHeader(404)
 	})
 

pkg/download/aur.go

@@ -1,18 +1,16 @@
 package download
 
 import (
-	"errors"
+	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/url"
 
-	"github.com/leonelquinteros/gotext"
+	"github.com/Jguer/yay/v10/pkg/settings/exe"
 )
 
 var AURPackageURL = "https://aur.archlinux.org/cgit/aur.git"
 
-var ErrAURPackageNotFound = errors.New(gotext.Get("package not found in AUR"))
-
 func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
 	values := url.Values{}
 	values.Set("h", pkgName)
@@ -22,8 +20,9 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	if resp.StatusCode != 200 {
+
-		return nil, ErrAURPackageNotFound
+	if resp.StatusCode != http.StatusOK {
+		return nil, ErrAURPackageNotFound{pkgName: pkgName}
 	}
 
 	defer resp.Body.Close()
@@ -35,3 +34,10 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) {
 
 	return pkgBuild, nil
 }
+
+// AURPkgbuildRepo retrieves the PKGBUILD repository to a dest directory.
+func AURPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, aurURL, pkgName, dest string, force bool) error {
+	pkgURL := fmt.Sprintf("%s/%s.git", aurURL, pkgName)
+
+	return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL, pkgName, dest, force)
+}

pkg/download/aur_test.go

@@ -10,13 +10,11 @@ import (
 
 func TestGetAURPkgbuild(t *testing.T) {
 	pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
 		w.WriteHeader(200)
 		w.Write([]byte(gitExtrasPKGBUILD))
 	})
 
 	notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-
 		w.WriteHeader(404)
 	})
 

pkg/download/errors.go

@@ -0,0 +1,31 @@
+package download
+
+import (
+	"fmt"
+
+	"github.com/leonelquinteros/gotext"
+)
+
+// ErrAURPackageNotFound means that package was not found in AUR.
+type ErrAURPackageNotFound struct {
+	pkgName string
+}
+
+func (e ErrAURPackageNotFound) Error() string {
+	return fmt.Sprintln(gotext.Get("package not found in AUR"), ":", e.pkgName)
+}
+
+type ErrGetPKGBUILDRepo struct {
+	inner   error
+	pkgName string
+	errOut  string
+}
+
+func (e ErrGetPKGBUILDRepo) Error() string {
+	return fmt.Sprintln(gotext.Get("error fetching %s: %s", e.pkgName, e.errOut),
+		"\n\t context:", e.inner.Error())
+}
+
+func (e *ErrGetPKGBUILDRepo) Unwrap() error {
+	return e.inner
+}

pkg/download/unified.go

@@ -2,14 +2,61 @@ package download
 
 import (
 	"net/http"
+	"os"
+	"path/filepath"
 	"sync"
 
+	"github.com/leonelquinteros/gotext"
+
+	"github.com/Jguer/go-alpm/v2"
+
 	"github.com/Jguer/yay/v10/pkg/db"
 	"github.com/Jguer/yay/v10/pkg/multierror"
 	"github.com/Jguer/yay/v10/pkg/settings"
+	"github.com/Jguer/yay/v10/pkg/settings/exe"
 	"github.com/Jguer/yay/v10/pkg/text"
 )
 
+func downloadGitRepo(cmdRunner exe.Runner,
+	cmdBuilder exe.GitCmdBuilder, pkgURL, pkgName, dest string, force bool, gitArgs ...string) error {
+	gitArgs = append(gitArgs, pkgURL, pkgName)
+
+	cloneArgs := make([]string, 0, len(gitArgs)+4)
+	cloneArgs = append(cloneArgs, "clone", "--no-progress")
+	cloneArgs = append(cloneArgs, gitArgs...)
+	finalDir := filepath.Join(dest, pkgName)
+
+	if _, err := os.Stat(filepath.Join(finalDir, ".git")); os.IsNotExist(err) {
+		if _, errD := os.Stat(finalDir); force && errD == nil {
+			if errR := os.RemoveAll(finalDir); errR != nil {
+				return ErrGetPKGBUILDRepo{inner: errR, pkgName: pkgName}
+			}
+		}
+
+		cmd := cmdBuilder.BuildGitCmd(dest, cloneArgs...)
+
+		_, stderr, errCapture := cmdRunner.Capture(cmd, 0)
+		if errCapture != nil {
+			return ErrGetPKGBUILDRepo{inner: errCapture, pkgName: pkgName, errOut: stderr}
+		}
+	} else if err != nil {
+		return ErrGetPKGBUILDRepo{
+			inner:   err,
+			pkgName: pkgName,
+			errOut:  gotext.Get("error reading %s", filepath.Join(dest, pkgName, ".git")),
+		}
+	} else {
+		cmd := cmdBuilder.BuildGitCmd(filepath.Join(dest, pkgName), "pull", "--ff-only")
+
+		_, stderr, errCmd := cmdRunner.Capture(cmd, 0)
+		if errCmd != nil {
+			return ErrGetPKGBUILDRepo{inner: errCmd, pkgName: pkgName, errOut: stderr}
+		}
+	}
+
+	return nil
+}
+
 func getURLName(pkg db.IPackage) string {
 	name := pkg.Base()
 	if name == "" {
@@ -21,13 +68,18 @@ func getURLName(pkg db.IPackage) string {
 
 func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []string, mode settings.TargetMode) (map[string][]byte, error) {
 	pkgbuilds := make(map[string][]byte, len(targets))
-	var mux sync.Mutex
+
-	var errs multierror.MultiError
+	var (
-	var wg sync.WaitGroup
+		mux  sync.Mutex
+		errs multierror.MultiError
+		wg   sync.WaitGroup
+	)
+
 	sem := make(chan uint8, MaxConcurrentFetch)
 
 	for _, target := range targets {
 		aur := true
+
 		dbName, name := text.SplitDBFromName(target)
 		if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) {
 			if pkg := dbExecutor.SyncPackage(name); pkg != nil {
@@ -43,11 +95,14 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str
 		}
 
 		sem <- 1
+
 		wg.Add(1)
 
 		go func(target, dbName, pkgName string, aur bool) {
-			var err error
+			var (
-			var pkgbuild []byte
+				err      error
+				pkgbuild []byte
+			)
 
 			if aur {
 				pkgbuild, err = GetAURPkgbuild(httpClient, pkgName)
@@ -72,3 +127,90 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str
 
 	return pkgbuilds, errs.Return()
 }
+
+func PKGBUILDRepos(dbExecutor db.Executor,
+	cmdRunner exe.Runner,
+	cmdBuilder exe.GitCmdBuilder,
+	targets []string, mode settings.TargetMode, aurURL, dest string, force bool) (map[string]bool, error) {
+	cloned := make(map[string]bool, len(targets))
+
+	var (
+		mux  sync.Mutex
+		errs multierror.MultiError
+		wg   sync.WaitGroup
+	)
+
+	sem := make(chan uint8, MaxConcurrentFetch)
+
+	for _, target := range targets {
+		aur := true
+
+		dbName, name := text.SplitDBFromName(target)
+		if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) {
+			var pkg alpm.IPackage
+			if dbName != "" {
+				pkg = dbExecutor.SatisfierFromDB(name, dbName)
+				if pkg == nil {
+					// if the user precised a db but the package is not in the db
+					// then it is missing
+					continue
+				}
+			} else {
+				pkg = dbExecutor.SyncPackage(name)
+			}
+
+			if pkg != nil {
+				aur = false
+				name = getURLName(pkg)
+				dbName = pkg.DB().Name()
+			}
+		}
+
+		if aur && mode == settings.ModeRepo {
+			// Mode does not allow AUR packages
+			continue
+		}
+
+		sem <- 1
+
+		wg.Add(1)
+
+		go func(target, dbName, pkgName string, aur bool) {
+			var (
+				err error
+			)
+
+			if aur {
+				err = AURPkgbuildRepo(cmdRunner, cmdBuilder, aurURL, pkgName, dest, force)
+			} else {
+				err = ABSPkgbuildRepo(cmdRunner, cmdBuilder, dbName, pkgName, dest, force)
+			}
+
+			success := err == nil
+			if success {
+				mux.Lock()
+				cloned[target] = success
+				mux.Unlock()
+			}
+			if !success {
+				errs.Add(err)
+			}
+
+			if aur {
+				text.OperationInfoln(
+					gotext.Get("(%d/%d) Downloaded PKGBUILD: %s",
+						len(cloned), len(targets), text.Cyan(pkgName)))
+			} else {
+				text.OperationInfoln(
+					gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s",
+						len(cloned), len(targets), text.Cyan(pkgName)))
+			}
+			<-sem
+			wg.Done()
+		}(target, dbName, name, aur)
+	}
+
+	wg.Wait()
+
+	return cloned, errs.Return()
+}

pkg/settings/exe/passers.go

@@ -5,6 +5,10 @@ import (
 	"os/exec"
 )
 
+type GitCmdBuilder interface {
+	BuildGitCmd(dir string, extraArgs ...string) *exec.Cmd
+}
+
 type CmdBuilder struct {
 	GitBin          string
 	GitFlags        []string

print.go

@@ -291,7 +291,9 @@ func printPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []s
 				missing = append(missing, target)
 			}
 		}
-		text.Warnln("Unable to find the following packages:", strings.Join(missing, ", "))
+		text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", "))
+
+		return fmt.Errorf("")
 	}
 
 	return nil