mirror of
https://github.com/Jguer/yay
synced 2024-08-31 21:58:39 +00:00
Merge pull request #641 from Morganamilo/parallel-G
Use goroutines for fetching packages via -G
This commit is contained in:
commit
19bf8e773c
|
@ -31,7 +31,8 @@ Perform yay specific print operations.
|
|||
|
||||
.TP
|
||||
.B \-G, \-\-getpkgbuild
|
||||
Downloads PKGBUILD from ABS or AUR.
|
||||
Downloads PKGBUILD from ABS or AUR. ABS pkgbuilds are always downloaded using
|
||||
tarballs and taken from trunk. The ABS can only be used for Arch Linux repositories
|
||||
|
||||
.RE
|
||||
If no arguments are provided 'yay \-Syu' will be performed.
|
||||
|
@ -276,7 +277,7 @@ Show a detailed list of updates in a similar format to VerbosePkgLists.
|
|||
Upgrades can also be skipped using numbers, number ranges or repo names.
|
||||
Adidionally ^ can be used to invert the selection.
|
||||
|
||||
\fBWarning\fR: It is not recommended to skip updates from the repositores as
|
||||
\fBWarning\fR: It is not recommended to skip updates from the repositories as
|
||||
this can lead to partial upgrades. This feature is intended to easily skip AUR
|
||||
updates on the fly that may be broken or have a long compile time. Ultimately
|
||||
it is up to the user what upgrades they skip.
|
||||
|
|
183
download.go
183
download.go
|
@ -8,6 +8,9 @@ import (
|
|||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
alpm "github.com/jguer/go-alpm"
|
||||
)
|
||||
|
||||
// Decide what download method to use:
|
||||
|
@ -103,10 +106,10 @@ func gitDiff(path string, name string) error {
|
|||
}
|
||||
|
||||
// DownloadAndUnpack downloads url tgz and extracts to path.
|
||||
func downloadAndUnpack(url string, path string) (err error) {
|
||||
err = os.MkdirAll(path, 0755)
|
||||
func downloadAndUnpack(url string, path string) error {
|
||||
err := os.MkdirAll(path, 0755)
|
||||
if err != nil {
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
fileName := filepath.Base(url)
|
||||
|
@ -116,15 +119,15 @@ func downloadAndUnpack(url string, path string) (err error) {
|
|||
|
||||
err = downloadFile(tarLocation, url)
|
||||
if err != nil {
|
||||
return
|
||||
return err
|
||||
}
|
||||
|
||||
err = exec.Command(config.TarBin, "-xf", tarLocation, "-C", path).Run()
|
||||
_, stderr, err := capture(exec.Command(config.TarBin, "-xf", tarLocation, "-C", path))
|
||||
if err != nil {
|
||||
return
|
||||
return fmt.Errorf("%s", stderr)
|
||||
}
|
||||
|
||||
return
|
||||
return nil
|
||||
}
|
||||
|
||||
func getPkgbuilds(pkgs []string) error {
|
||||
|
@ -135,9 +138,18 @@ func getPkgbuilds(pkgs []string) error {
|
|||
}
|
||||
|
||||
pkgs = removeInvalidTargets(pkgs)
|
||||
|
||||
aur, repo, err := packageSlices(pkgs)
|
||||
|
||||
for n := range aur {
|
||||
_, pkg := splitDbFromName(aur[n])
|
||||
aur[n] = pkg
|
||||
}
|
||||
|
||||
info, err := aurInfoPrint(aur)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(repo) > 0 {
|
||||
missing, err = getPkgbuildsfromABS(repo, wd)
|
||||
if err != nil {
|
||||
|
@ -146,11 +158,13 @@ func getPkgbuilds(pkgs []string) error {
|
|||
}
|
||||
|
||||
if len(aur) > 0 {
|
||||
_missing, err := getPkgbuildsfromAUR(aur, wd)
|
||||
if err != nil {
|
||||
bases := getBases(info)
|
||||
toSkip := pkgbuildsToSkip(bases, nil)
|
||||
if _, err = downloadPkgbuilds(bases, toSkip, wd); err != nil {
|
||||
return err
|
||||
}
|
||||
missing = missing || _missing
|
||||
|
||||
missing = missing || len(aur) != len(info)
|
||||
}
|
||||
|
||||
if missing {
|
||||
|
@ -161,107 +175,94 @@ func getPkgbuilds(pkgs []string) error {
|
|||
}
|
||||
|
||||
// GetPkgbuild downloads pkgbuild from the ABS.
|
||||
func getPkgbuildsfromABS(pkgs []string, path string) (missing bool, err error) {
|
||||
func getPkgbuildsfromABS(pkgs []string, path string) (bool, error) {
|
||||
var wg sync.WaitGroup
|
||||
var mux sync.Mutex
|
||||
var errs MultiError
|
||||
names := make(map[string]string)
|
||||
missing := make([]string, 0)
|
||||
downloaded := 0
|
||||
|
||||
dbList, err := alpmHandle.SyncDbs()
|
||||
if err != nil {
|
||||
return
|
||||
return false, err
|
||||
}
|
||||
|
||||
nextPkg:
|
||||
for _, pkgN := range pkgs {
|
||||
var pkg *alpm.Package
|
||||
var err error
|
||||
var url string
|
||||
pkgDb, name := splitDbFromName(pkgN)
|
||||
|
||||
for _, db := range dbList.Slice() {
|
||||
if pkgDb != "" && db.Name() != pkgDb {
|
||||
continue
|
||||
if pkgDb != "" {
|
||||
if db, err := alpmHandle.SyncDbByName(pkgDb); err == nil {
|
||||
pkg, err = db.PkgByName(name)
|
||||
}
|
||||
|
||||
pkg, err := db.PkgByName(name)
|
||||
if err == nil {
|
||||
var url string
|
||||
name := pkg.Base()
|
||||
if name == "" {
|
||||
name = pkg.Name()
|
||||
} else {
|
||||
dbList.ForEach(func(db alpm.Db) error {
|
||||
if pkg, err = db.PkgByName(name); err == nil {
|
||||
return fmt.Errorf("")
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(path, name)); err == nil {
|
||||
fmt.Println(bold(red(arrow)), bold(cyan(name)), "directory already exists")
|
||||
continue nextPkg
|
||||
}
|
||||
|
||||
switch db.Name() {
|
||||
case "core", "extra":
|
||||
url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
|
||||
case "community", "multilib":
|
||||
url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
|
||||
default:
|
||||
fmt.Println(pkgN, "not in standard repositories")
|
||||
continue nextPkg
|
||||
}
|
||||
|
||||
errD := downloadAndUnpack(url, cacheHome)
|
||||
if errD != nil {
|
||||
fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
|
||||
}
|
||||
|
||||
errD = exec.Command("mv", filepath.Join(cacheHome, "packages", name, "trunk"), filepath.Join(path, name)).Run()
|
||||
if errD != nil {
|
||||
fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name())), bold(red(errD.Error())))
|
||||
} else {
|
||||
fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.Name()), "from ABS")
|
||||
}
|
||||
|
||||
continue nextPkg
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
fmt.Println(pkgN, "could not find package in database")
|
||||
missing = true
|
||||
}
|
||||
|
||||
if _, err := os.Stat(filepath.Join(cacheHome, "packages")); err == nil {
|
||||
os.RemoveAll(filepath.Join(cacheHome, "packages"))
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// GetPkgbuild downloads pkgbuild from the AUR.
|
||||
func getPkgbuildsfromAUR(pkgs []string, dir string) (bool, error) {
|
||||
missing := false
|
||||
strippedPkgs := make([]string, 0)
|
||||
for _, pkg := range pkgs {
|
||||
_, name := splitDbFromName(pkg)
|
||||
strippedPkgs = append(strippedPkgs, name)
|
||||
}
|
||||
|
||||
aq, err := aurInfoPrint(strippedPkgs)
|
||||
if err != nil {
|
||||
return missing, err
|
||||
}
|
||||
|
||||
for _, pkg := range aq {
|
||||
if _, err := os.Stat(filepath.Join(dir, pkg.PackageBase)); err == nil {
|
||||
fmt.Println(bold(red(arrow)), bold(cyan(pkg.Name)), "directory already exists")
|
||||
if pkg == nil {
|
||||
missing = append(missing, name)
|
||||
continue
|
||||
}
|
||||
|
||||
if shouldUseGit(filepath.Join(dir, pkg.PackageBase)) {
|
||||
_, err = gitDownload(baseURL+"/"+pkg.PackageBase+".git", dir, pkg.PackageBase)
|
||||
} else {
|
||||
err = downloadAndUnpack(baseURL+aq[0].URLPath, dir)
|
||||
name = pkg.Base()
|
||||
if name == "" {
|
||||
name = pkg.Name()
|
||||
}
|
||||
|
||||
switch pkg.DB().Name() {
|
||||
case "core", "extra", "testing":
|
||||
url = "https://git.archlinux.org/svntogit/packages.git/snapshot/packages/" + name + ".tar.gz"
|
||||
case "community", "multilib", "community-testing", "multilib-testing":
|
||||
url = "https://git.archlinux.org/svntogit/community.git/snapshot/packages/" + name + ".tar.gz"
|
||||
default:
|
||||
missing = append(missing, name)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = os.RemoveAll(filepath.Join(path, name)); err != nil {
|
||||
fmt.Println(bold(red(smallArrow)), err)
|
||||
continue
|
||||
}
|
||||
|
||||
names[name] = url
|
||||
}
|
||||
|
||||
if len(missing) != 0 {
|
||||
fmt.Println(yellow(bold(smallArrow)), "Missing ABS packages: ", cyan(strings.Join(missing, " ")))
|
||||
}
|
||||
|
||||
download := func(pkg string, url string) {
|
||||
defer wg.Done()
|
||||
if err := downloadAndUnpack(url, cacheHome); err != nil {
|
||||
errs.Add(fmt.Errorf("%s Failed to get pkgbuild: %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(err.Error()))))
|
||||
return
|
||||
}
|
||||
|
||||
_, stderr, err := capture(exec.Command("mv", filepath.Join(cacheHome, "packages", pkg, "trunk"), filepath.Join(path, pkg)))
|
||||
mux.Lock()
|
||||
downloaded++
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
errs.Add(fmt.Errorf("%s Failed to move %s: %s", bold(red(arrow)), bold(cyan(pkg)), bold(red(string(stderr)))))
|
||||
} else {
|
||||
fmt.Println(bold(yellow(arrow)), "Downloaded", cyan(pkg.PackageBase), "from AUR")
|
||||
fmt.Printf(bold(cyan("::"))+" Downloaded PKGBUILD from ABS (%d/%d): %s\n", downloaded, len(names), cyan(pkg))
|
||||
}
|
||||
mux.Unlock()
|
||||
}
|
||||
|
||||
if len(aq) != len(pkgs) {
|
||||
missing = true
|
||||
for name, url := range names {
|
||||
wg.Add(1)
|
||||
go download(name, url)
|
||||
}
|
||||
|
||||
return missing, err
|
||||
wg.Wait()
|
||||
errs.Add(os.RemoveAll(filepath.Join(cacheHome, "packages")))
|
||||
return len(missing) != 0, errs.Return()
|
||||
}
|
||||
|
|
4
exec.go
4
exec.go
|
@ -25,8 +25,8 @@ func capture(cmd *exec.Cmd) (string, string, error) {
|
|||
cmd.Stdout = &outbuf
|
||||
cmd.Stderr = &errbuf
|
||||
err := cmd.Run()
|
||||
stdout := outbuf.String()
|
||||
stderr := errbuf.String()
|
||||
stdout := strings.TrimSpace(outbuf.String())
|
||||
stderr := strings.TrimSpace(errbuf.String())
|
||||
|
||||
return stdout, stderr, err
|
||||
}
|
||||
|
|
|
@ -185,7 +185,7 @@ func install(parser *arguments) error {
|
|||
}
|
||||
|
||||
toSkip := pkgbuildsToSkip(do.Aur, targets)
|
||||
cloned, err := downloadPkgbuilds(do.Aur, toSkip)
|
||||
cloned, err := downloadPkgbuilds(do.Aur, toSkip, config.BuildDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -809,7 +809,7 @@ func mergePkgbuilds(bases []Base) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
|
||||
func downloadPkgbuilds(bases []Base, toSkip stringSet, buildDir string) (stringSet, error) {
|
||||
cloned := make(stringSet)
|
||||
downloaded := 0
|
||||
var wg sync.WaitGroup
|
||||
|
@ -830,7 +830,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
|
|||
}
|
||||
|
||||
if shouldUseGit(filepath.Join(config.BuildDir, pkg)) {
|
||||
clone, err := gitDownload(baseURL+"/"+pkg+".git", config.BuildDir, pkg)
|
||||
clone, err := gitDownload(baseURL+"/"+pkg+".git", buildDir, pkg)
|
||||
if err != nil {
|
||||
errs.Add(err)
|
||||
return
|
||||
|
@ -841,7 +841,7 @@ func downloadPkgbuilds(bases []Base, toSkip stringSet) (stringSet, error) {
|
|||
mux.Unlock()
|
||||
}
|
||||
} else {
|
||||
err := downloadAndUnpack(baseURL+base.URLPath(), config.BuildDir)
|
||||
err := downloadAndUnpack(baseURL+base.URLPath(), buildDir)
|
||||
if err != nil {
|
||||
errs.Add(err)
|
||||
return
|
||||
|
|
4
utils.go
4
utils.go
|
@ -139,10 +139,10 @@ func (err *MultiError) Error() string {
|
|||
str := ""
|
||||
|
||||
for _, e := range err.Errors {
|
||||
str += e.Error()
|
||||
str += e.Error() + "\n"
|
||||
}
|
||||
|
||||
return str
|
||||
return str[:len(str)-1]
|
||||
}
|
||||
|
||||
func (err *MultiError) Add(e error) {
|
||||
|
|
2
vcs.go
2
vcs.go
|
@ -38,7 +38,7 @@ func createDevelDB() error {
|
|||
|
||||
bases := getBases(info)
|
||||
toSkip := pkgbuildsToSkip(bases, sliceToStringSet(remoteNames))
|
||||
downloadPkgbuilds(bases, toSkip)
|
||||
downloadPkgbuilds(bases, toSkip, config.BuildDir)
|
||||
srcinfos, _ := parseSrcinfoFiles(bases, false)
|
||||
|
||||
for _, pkgbuild := range srcinfos {
|
||||
|
|
Loading…
Reference in a new issue