From 79b44fd5448a15919aaa3c06e5f98cc0bd0d19ba Mon Sep 17 00:00:00 2001 From: jguer Date: Mon, 2 Aug 2021 18:00:50 +0200 Subject: [PATCH] feat(download): download PKGBUILD repos interface --- .golangci.yml | 4 +- cmd.go | 4 +- download.go | 209 ++++-------------------------------- pkg/download/abs.go | 48 +++++++-- pkg/download/abs_test.go | 2 - pkg/download/aur.go | 18 ++-- pkg/download/aur_test.go | 2 - pkg/download/errors.go | 31 ++++++ pkg/download/unified.go | 152 +++++++++++++++++++++++++- pkg/settings/exe/passers.go | 4 + print.go | 4 +- 11 files changed, 258 insertions(+), 220 deletions(-) create mode 100644 pkg/download/errors.go diff --git a/.golangci.yml b/.golangci.yml index 3eef7d17..a827d97b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -54,7 +54,6 @@ linters: - gocritic - gofmt - goimports - - golint - goprintffuncname - gosec - gosimple @@ -63,6 +62,8 @@ linters: - lll - misspell - nakedret + - prealloc + - revive - rowserrcheck - staticcheck - structcheck @@ -73,7 +74,6 @@ linters: - unused - varcheck - whitespace - - prealloc # disabled want to fix #- scopelint diff --git a/cmd.go b/cmd.go index 0aa5cfd2..4ab54f37 100644 --- a/cmd.go +++ b/cmd.go @@ -277,7 +277,9 @@ func handleGetpkgbuild(cmdArgs *settings.Arguments, dbExecutor db.Executor) erro if cmdArgs.ExistsArg("p", "print") { return printPkgbuilds(dbExecutor, config.Runtime.HTTPClient, cmdArgs.Targets) } - return getPkgbuilds(cmdArgs.Targets, dbExecutor, cmdArgs.ExistsArg("f", "force")) + return getPkgbuilds(dbExecutor, + config.Runtime.CmdRunner, config.Runtime.CmdBuilder, + cmdArgs.Targets, config.Runtime.Mode, config.AURURL, cmdArgs.ExistsArg("f", "force")) } func handleYogurt(cmdArgs *settings.Arguments, dbExecutor db.Executor) error { diff --git a/download.go b/download.go index c1e67b28..f7028f2f 100644 --- a/download.go +++ b/download.go @@ -3,20 +3,15 @@ package main import ( "fmt" "os" - "os/exec" "path/filepath" "strings" - "sync" "github.com/leonelquinteros/gotext" - "github.com/pkg/errors" - - alpm "github.com/Jguer/go-alpm/v2" "github.com/Jguer/yay/v10/pkg/db" - "github.com/Jguer/yay/v10/pkg/dep" - "github.com/Jguer/yay/v10/pkg/multierror" - "github.com/Jguer/yay/v10/pkg/query" + "github.com/Jguer/yay/v10/pkg/download" + "github.com/Jguer/yay/v10/pkg/settings" + "github.com/Jguer/yay/v10/pkg/settings/exe" "github.com/Jguer/yay/v10/pkg/text" ) @@ -80,34 +75,6 @@ func gitHasDiff(path, name string) (bool, error) { return true, nil } -// TODO: yay-next passes args through the header, use that to unify ABS and AUR -func gitDownloadABS(url, path, name string) (bool, error) { - if err := os.MkdirAll(path, 0o755); err != nil { - return false, err - } - - if _, errExist := os.Stat(filepath.Join(path, name)); os.IsNotExist(errExist) { - cmd := config.Runtime.CmdBuilder.BuildGitCmd(path, "clone", "--no-progress", "--single-branch", - "-b", "packages/"+name, url, name) - _, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0) - if err != nil { - return false, fmt.Errorf(gotext.Get("error cloning %s: %s", name, stderr)) - } - - return true, nil - } else if errExist != nil { - return false, fmt.Errorf(gotext.Get("error reading %s", filepath.Join(path, name, ".git"))) - } - - cmd := config.Runtime.CmdBuilder.BuildGitCmd(filepath.Join(path, name), "pull", "--ff-only") - _, stderr, err := config.Runtime.CmdRunner.Capture(cmd, 0) - if err != nil { - return false, fmt.Errorf(gotext.Get("error fetching %s: %s", name, stderr)) - } - - return true, nil -} - func gitDownload(url, path, name string) (bool, error) { _, err := os.Stat(filepath.Join(path, name, ".git")) if os.IsNotExist(err) { @@ -149,173 +116,33 @@ func gitMerge(path, name string) error { return nil } -func getPkgbuilds(pkgs []string, dbExecutor db.Executor, force bool) error { - missing := false +func getPkgbuilds(dbExecutor db.Executor, + cmdRunner exe.Runner, + cmdBuilder exe.GitCmdBuilder, targets []string, + mode settings.TargetMode, + aurURL string, + force bool) error { wd, err := os.Getwd() if err != nil { return err } - pkgs = query.RemoveInvalidTargets(pkgs, config.Runtime.Mode) - aur, repo := packageSlices(pkgs, dbExecutor) - - for n := range aur { - _, pkg := text.SplitDBFromName(aur[n]) - aur[n] = pkg + cloned, errD := download.PKGBUILDRepos(dbExecutor, cmdRunner, cmdBuilder, targets, mode, aurURL, wd, force) + if errD != nil { + text.Errorln(errD) } - info, err := query.AURInfoPrint(config.Runtime.AURClient, aur, config.RequestSplitN) - if err != nil { - return err - } - - if len(repo) > 0 { - missing, err = getPkgbuildsfromABS(repo, wd, dbExecutor, force) - if err != nil { - return err - } - } - - if len(aur) > 0 { - allBases := dep.GetBases(info) - bases := make([]dep.Base, 0) - - for _, base := range allBases { - name := base.Pkgbase() - pkgDest := filepath.Join(wd, name) - _, err = os.Stat(pkgDest) - if os.IsNotExist(err) { - bases = append(bases, base) - } else if err != nil { - text.Errorln(err) - continue - } else { - if force { - if err = os.RemoveAll(pkgDest); err != nil { - text.Errorln(err) - continue - } - bases = append(bases, base) - } else { - text.Warnln(gotext.Get("%s already exists. Use -f/--force to overwrite", pkgDest)) - continue - } + if len(targets) != len(cloned) { + missing := []string{} + for _, target := range targets { + if _, ok := cloned[target]; !ok { + missing = append(missing, target) } } + text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", ")) - if _, err = downloadPkgbuilds(bases, nil, wd); err != nil { - return err - } - - missing = missing || len(aur) != len(info) - } - - if missing { err = fmt.Errorf("") } return err } - -// GetPkgbuild downloads pkgbuild from the ABS. -func getPkgbuildsfromABS(pkgs []string, path string, dbExecutor db.Executor, force bool) (bool, error) { - var wg sync.WaitGroup - var mux sync.Mutex - var errs multierror.MultiError - names := make(map[string]string) - missing := make([]string, 0) - downloaded := 0 - - for _, pkgN := range pkgs { - var pkg alpm.IPackage - var err error - var url string - pkgDB, name := text.SplitDBFromName(pkgN) - - if pkgDB != "" { - pkg = dbExecutor.SatisfierFromDB(name, pkgDB) - } else { - pkg = dbExecutor.SyncSatisfier(name) - } - - if pkg == nil { - missing = append(missing, name) - continue - } - - name = pkg.Base() - if name == "" { - name = pkg.Name() - } - - // TODO: Check existence with ls-remote - // https://git.archlinux.org/svntogit/packages.git - switch pkg.DB().Name() { - case "core", "extra", "testing": - url = "https://github.com/archlinux/svntogit-packages.git" - case "community", "multilib", "community-testing", "multilib-testing": - url = "https://github.com/archlinux/svntogit-community.git" - default: - missing = append(missing, name) - continue - } - - _, err = os.Stat(filepath.Join(path, name)) - switch { - case err != nil && !os.IsNotExist(err): - text.Errorln(err) - continue - case os.IsNotExist(err), force: - if err = os.RemoveAll(filepath.Join(path, name)); err != nil { - text.Errorln(err) - continue - } - default: - text.Warn(gotext.Get("%s already downloaded -- use -f to overwrite", text.Cyan(name))) - continue - } - - names[name] = url - } - - if len(missing) != 0 { - text.Warnln(gotext.Get("Missing ABS packages:"), - text.Cyan(strings.Join(missing, ", "))) - } - - download := func(pkg string, url string) { - defer wg.Done() - if _, err := gitDownloadABS(url, config.ABSDir, pkg); err != nil { - errs.Add(errors.New(gotext.Get("failed to get pkgbuild: %s: %s", text.Cyan(pkg), err.Error()))) - return - } - - _, stderr, err := config.Runtime.CmdRunner.Capture( - exec.Command( - "cp", "-r", - filepath.Join(config.ABSDir, pkg, "trunk"), - filepath.Join(path, pkg)), 0) - mux.Lock() - downloaded++ - if err != nil { - errs.Add(errors.New(gotext.Get("failed to link %s: %s", text.Cyan(pkg), stderr))) - } else { - fmt.Fprintln(os.Stdout, gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s", downloaded, len(names), text.Cyan(pkg))) - } - mux.Unlock() - } - - count := 0 - for name, url := range names { - wg.Add(1) - go download(name, url) - count++ - if count%25 == 0 { - wg.Wait() - } - } - - wg.Wait() - - return len(missing) != 0, errs.Return() -} diff --git a/pkg/download/abs.go b/pkg/download/abs.go index 31c8987c..a4ee9409 100644 --- a/pkg/download/abs.go +++ b/pkg/download/abs.go @@ -7,6 +7,8 @@ import ( "net/http" "github.com/leonelquinteros/gotext" + + "github.com/Jguer/yay/v10/pkg/settings/exe" ) const ( @@ -21,22 +23,37 @@ var ( ABSCommunityURL = "https://github.com/archlinux/svntogit-community" ) +func getRepoURL(db string) (string, error) { + switch db { + case "core", "extra", "testing": + return ABSPackageURL, nil + case "community", "multilib", "community-testing", "multilib-testing": + return ABSCommunityURL, nil + } + + return "", ErrInvalidRepository +} + // Return format for pkgbuild // https://github.com/archlinux/svntogit-community/raw/packages/neovim/trunk/PKGBUILD func getPackageURL(db, pkgName string) (string, error) { - repoURL := "" - switch db { - case "core", "extra", "testing": - repoURL = ABSPackageURL - case "community", "multilib", "community-testing", "multilib-testing": - repoURL = ABSCommunityURL - default: - return "", ErrInvalidRepository + repoURL, err := getRepoURL(db) + if err != nil { + return "", err } - return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), nil + return fmt.Sprintf(_urlPackagePath, repoURL, pkgName), err } +// Return format for pkgbuild repo +// https://github.com/archlinux/svntogit-community.git +func getPackageRepoURL(db string) (string, error) { + repoURL, err := getRepoURL(db) + + return repoURL + ".git", err +} + +// GetABSPkgbuild retrieves the PKGBUILD file to a dest directory func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, error) { packageURL, err := getPackageURL(dbName, pkgName) if err != nil { @@ -48,7 +65,7 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er return nil, err } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { return nil, ErrABSPackageNotFound } @@ -61,3 +78,14 @@ func GetABSPkgbuild(httpClient *http.Client, dbName, pkgName string) ([]byte, er return pkgBuild, nil } + +// ABSPkgbuildRepo retrieves the PKGBUILD repository to a dest directory +func ABSPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, dbName, pkgName, dest string, force bool) error { + pkgURL, err := getPackageRepoURL(dbName) + if err != nil { + return err + } + + return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL, + pkgName, dest, force, "--single-branch", "-b", "packages/"+pkgName) +} diff --git a/pkg/download/abs_test.go b/pkg/download/abs_test.go index a0c102c4..f616cab8 100644 --- a/pkg/download/abs_test.go +++ b/pkg/download/abs_test.go @@ -81,13 +81,11 @@ func Test_getPackageURL(t *testing.T) { func TestGetABSPkgbuild(t *testing.T) { pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) w.Write([]byte(gitExtrasPKGBUILD)) }) notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(404) }) diff --git a/pkg/download/aur.go b/pkg/download/aur.go index 4cc7a324..2470f070 100644 --- a/pkg/download/aur.go +++ b/pkg/download/aur.go @@ -1,18 +1,16 @@ package download import ( - "errors" + "fmt" "io/ioutil" "net/http" "net/url" - "github.com/leonelquinteros/gotext" + "github.com/Jguer/yay/v10/pkg/settings/exe" ) var AURPackageURL = "https://aur.archlinux.org/cgit/aur.git" -var ErrAURPackageNotFound = errors.New(gotext.Get("package not found in AUR")) - func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) { values := url.Values{} values.Set("h", pkgName) @@ -22,8 +20,9 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) { if err != nil { return nil, err } - if resp.StatusCode != 200 { - return nil, ErrAURPackageNotFound + + if resp.StatusCode != http.StatusOK { + return nil, ErrAURPackageNotFound{pkgName: pkgName} } defer resp.Body.Close() @@ -35,3 +34,10 @@ func GetAURPkgbuild(httpClient *http.Client, pkgName string) ([]byte, error) { return pkgBuild, nil } + +// AURPkgbuildRepo retrieves the PKGBUILD repository to a dest directory. +func AURPkgbuildRepo(cmdRunner exe.Runner, cmdBuilder exe.GitCmdBuilder, aurURL, pkgName, dest string, force bool) error { + pkgURL := fmt.Sprintf("%s/%s.git", aurURL, pkgName) + + return downloadGitRepo(cmdRunner, cmdBuilder, pkgURL, pkgName, dest, force) +} diff --git a/pkg/download/aur_test.go b/pkg/download/aur_test.go index dfc7078e..7d1970e9 100644 --- a/pkg/download/aur_test.go +++ b/pkg/download/aur_test.go @@ -10,13 +10,11 @@ import ( func TestGetAURPkgbuild(t *testing.T) { pkgBuildHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) w.Write([]byte(gitExtrasPKGBUILD)) }) notFoundHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(404) }) diff --git a/pkg/download/errors.go b/pkg/download/errors.go new file mode 100644 index 00000000..72b62d74 --- /dev/null +++ b/pkg/download/errors.go @@ -0,0 +1,31 @@ +package download + +import ( + "fmt" + + "github.com/leonelquinteros/gotext" +) + +// ErrAURPackageNotFound means that package was not found in AUR. +type ErrAURPackageNotFound struct { + pkgName string +} + +func (e ErrAURPackageNotFound) Error() string { + return fmt.Sprintln(gotext.Get("package not found in AUR"), ":", e.pkgName) +} + +type ErrGetPKGBUILDRepo struct { + inner error + pkgName string + errOut string +} + +func (e ErrGetPKGBUILDRepo) Error() string { + return fmt.Sprintln(gotext.Get("error fetching %s: %s", e.pkgName, e.errOut), + "\n\t context:", e.inner.Error()) +} + +func (e *ErrGetPKGBUILDRepo) Unwrap() error { + return e.inner +} diff --git a/pkg/download/unified.go b/pkg/download/unified.go index e6cc4b2c..15ae0708 100644 --- a/pkg/download/unified.go +++ b/pkg/download/unified.go @@ -2,14 +2,61 @@ package download import ( "net/http" + "os" + "path/filepath" "sync" + "github.com/leonelquinteros/gotext" + + "github.com/Jguer/go-alpm/v2" + "github.com/Jguer/yay/v10/pkg/db" "github.com/Jguer/yay/v10/pkg/multierror" "github.com/Jguer/yay/v10/pkg/settings" + "github.com/Jguer/yay/v10/pkg/settings/exe" "github.com/Jguer/yay/v10/pkg/text" ) +func downloadGitRepo(cmdRunner exe.Runner, + cmdBuilder exe.GitCmdBuilder, pkgURL, pkgName, dest string, force bool, gitArgs ...string) error { + gitArgs = append(gitArgs, pkgURL, pkgName) + + cloneArgs := make([]string, 0, len(gitArgs)+4) + cloneArgs = append(cloneArgs, "clone", "--no-progress") + cloneArgs = append(cloneArgs, gitArgs...) + finalDir := filepath.Join(dest, pkgName) + + if _, err := os.Stat(filepath.Join(finalDir, ".git")); os.IsNotExist(err) { + if _, errD := os.Stat(finalDir); force && errD == nil { + if errR := os.RemoveAll(finalDir); errR != nil { + return ErrGetPKGBUILDRepo{inner: errR, pkgName: pkgName} + } + } + + cmd := cmdBuilder.BuildGitCmd(dest, cloneArgs...) + + _, stderr, errCapture := cmdRunner.Capture(cmd, 0) + if errCapture != nil { + return ErrGetPKGBUILDRepo{inner: errCapture, pkgName: pkgName, errOut: stderr} + } + } else if err != nil { + return ErrGetPKGBUILDRepo{ + inner: err, + pkgName: pkgName, + errOut: gotext.Get("error reading %s", filepath.Join(dest, pkgName, ".git")), + } + } else { + cmd := cmdBuilder.BuildGitCmd(filepath.Join(dest, pkgName), "pull", "--ff-only") + + _, stderr, errCmd := cmdRunner.Capture(cmd, 0) + if errCmd != nil { + return ErrGetPKGBUILDRepo{inner: errCmd, pkgName: pkgName, errOut: stderr} + } + } + + return nil +} + func getURLName(pkg db.IPackage) string { name := pkg.Base() if name == "" { @@ -21,13 +68,18 @@ func getURLName(pkg db.IPackage) string { func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []string, mode settings.TargetMode) (map[string][]byte, error) { pkgbuilds := make(map[string][]byte, len(targets)) - var mux sync.Mutex - var errs multierror.MultiError - var wg sync.WaitGroup + + var ( + mux sync.Mutex + errs multierror.MultiError + wg sync.WaitGroup + ) + sem := make(chan uint8, MaxConcurrentFetch) for _, target := range targets { aur := true + dbName, name := text.SplitDBFromName(target) if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) { if pkg := dbExecutor.SyncPackage(name); pkg != nil { @@ -43,11 +95,14 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str } sem <- 1 + wg.Add(1) go func(target, dbName, pkgName string, aur bool) { - var err error - var pkgbuild []byte + var ( + err error + pkgbuild []byte + ) if aur { pkgbuild, err = GetAURPkgbuild(httpClient, pkgName) @@ -72,3 +127,90 @@ func GetPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []str return pkgbuilds, errs.Return() } + +func PKGBUILDRepos(dbExecutor db.Executor, + cmdRunner exe.Runner, + cmdBuilder exe.GitCmdBuilder, + targets []string, mode settings.TargetMode, aurURL, dest string, force bool) (map[string]bool, error) { + cloned := make(map[string]bool, len(targets)) + + var ( + mux sync.Mutex + errs multierror.MultiError + wg sync.WaitGroup + ) + + sem := make(chan uint8, MaxConcurrentFetch) + + for _, target := range targets { + aur := true + + dbName, name := text.SplitDBFromName(target) + if dbName != "aur" && (mode == settings.ModeAny || mode == settings.ModeRepo) { + var pkg alpm.IPackage + if dbName != "" { + pkg = dbExecutor.SatisfierFromDB(name, dbName) + if pkg == nil { + // if the user precised a db but the package is not in the db + // then it is missing + continue + } + } else { + pkg = dbExecutor.SyncPackage(name) + } + + if pkg != nil { + aur = false + name = getURLName(pkg) + dbName = pkg.DB().Name() + } + } + + if aur && mode == settings.ModeRepo { + // Mode does not allow AUR packages + continue + } + + sem <- 1 + + wg.Add(1) + + go func(target, dbName, pkgName string, aur bool) { + var ( + err error + ) + + if aur { + err = AURPkgbuildRepo(cmdRunner, cmdBuilder, aurURL, pkgName, dest, force) + } else { + err = ABSPkgbuildRepo(cmdRunner, cmdBuilder, dbName, pkgName, dest, force) + } + + success := err == nil + if success { + mux.Lock() + cloned[target] = success + mux.Unlock() + } + if !success { + errs.Add(err) + } + + if aur { + text.OperationInfoln( + gotext.Get("(%d/%d) Downloaded PKGBUILD: %s", + len(cloned), len(targets), text.Cyan(pkgName))) + } else { + text.OperationInfoln( + gotext.Get("(%d/%d) Downloaded PKGBUILD from ABS: %s", + len(cloned), len(targets), text.Cyan(pkgName))) + } + <-sem + wg.Done() + }(target, dbName, name, aur) + } + + wg.Wait() + + return cloned, errs.Return() +} diff --git a/pkg/settings/exe/passers.go b/pkg/settings/exe/passers.go index 30c7aa45..8fed3197 100644 --- a/pkg/settings/exe/passers.go +++ b/pkg/settings/exe/passers.go @@ -5,6 +5,10 @@ import ( "os/exec" ) +type GitCmdBuilder interface { + BuildGitCmd(dir string, extraArgs ...string) *exec.Cmd +} + type CmdBuilder struct { GitBin string GitFlags []string diff --git a/print.go b/print.go index 01ecf5f1..ac1279b9 100644 --- a/print.go +++ b/print.go @@ -291,7 +291,9 @@ func printPkgbuilds(dbExecutor db.Executor, httpClient *http.Client, targets []s missing = append(missing, target) } } - text.Warnln("Unable to find the following packages:", strings.Join(missing, ", ")) + text.Warnln(gotext.Get("Unable to find the following packages:"), strings.Join(missing, ", ")) + + return fmt.Errorf("") } return nil