yay/query.go

580 lines
12 KiB
Go
Raw Normal View History

2017-04-29 17:12:12 +00:00
package main
import (
"errors"
2017-04-29 17:12:12 +00:00
"fmt"
"os"
"sort"
"strings"
2018-02-19 04:25:36 +00:00
"sync"
"time"
2017-04-29 17:12:12 +00:00
2019-04-23 15:53:20 +00:00
alpm "github.com/Jguer/go-alpm"
"github.com/leonelquinteros/gotext"
rpc "github.com/mikkeloscar/aur"
"github.com/Jguer/yay/v10/pkg/intrange"
"github.com/Jguer/yay/v10/pkg/multierror"
"github.com/Jguer/yay/v10/pkg/stringset"
"github.com/Jguer/yay/v10/pkg/text"
2017-04-29 17:12:12 +00:00
)
type aurWarnings struct {
Orphans []string
OutOfDate []string
Missing []string
2019-11-11 07:15:04 +00:00
Ignore stringset.StringSet
}
func makeWarnings() *aurWarnings {
return &aurWarnings{Ignore: make(stringset.StringSet)}
}
// Query is a collection of Results
type aurQuery []rpc.Pkg
// Query holds the results of a repository search.
type repoQuery []alpm.Package
func (q aurQuery) Len() int {
return len(q)
}
func (q aurQuery) Less(i, j int) bool {
var result bool
switch config.SortBy {
case "votes":
result = q[i].NumVotes > q[j].NumVotes
case "popularity":
result = q[i].Popularity > q[j].Popularity
case "name":
result = LessRunes([]rune(q[i].Name), []rune(q[j].Name))
case "base":
result = LessRunes([]rune(q[i].PackageBase), []rune(q[j].PackageBase))
case "submitted":
result = q[i].FirstSubmitted < q[j].FirstSubmitted
case "modified":
result = q[i].LastModified < q[j].LastModified
case "id":
result = q[i].ID < q[j].ID
case "baseid":
result = q[i].PackageBaseID < q[j].PackageBaseID
}
if config.SortMode == bottomUp {
return !result
}
return result
}
func (q aurQuery) Swap(i, j int) {
q[i], q[j] = q[j], q[i]
}
// FilterPackages filters packages based on source and type from local repository.
func filterPackages() (
local, remote []alpm.Package,
localNames, remoteNames []string,
err error) {
2019-02-04 16:56:02 +00:00
localDB, err := alpmHandle.LocalDB()
if err != nil {
return
}
2019-02-04 16:56:02 +00:00
dbList, err := alpmHandle.SyncDBs()
if err != nil {
return
}
f := func(k alpm.Package) error {
found := false
// For each DB search for our secret package.
2019-02-04 16:56:02 +00:00
_ = dbList.ForEach(func(d alpm.DB) error {
if found {
return nil
}
2019-02-14 20:44:39 +00:00
if d.Pkg(k.Name()) != nil {
found = true
local = append(local, k)
localNames = append(localNames, k.Name())
}
return nil
})
if !found {
remote = append(remote, k)
remoteNames = append(remoteNames, k.Name())
}
return nil
}
2019-02-04 16:56:02 +00:00
err = localDB.PkgCache().ForEach(f)
return local, remote, localNames, remoteNames, err
}
func getSearchBy(value string) rpc.By {
switch value {
case "name":
return rpc.Name
case "maintainer":
return rpc.Maintainer
case "depends":
return rpc.Depends
case "makedepends":
return rpc.MakeDepends
case "optdepends":
return rpc.OptDepends
case "checkdepends":
return rpc.CheckDepends
default:
return rpc.NameDesc
}
}
// NarrowSearch searches AUR and narrows based on subarguments
func narrowSearch(pkgS []string, sortS bool) (aurQuery, error) {
var r []rpc.Pkg
var err error
var usedIndex int
by := getSearchBy(config.SearchBy)
if len(pkgS) == 0 {
return nil, nil
}
for i, word := range pkgS {
r, err = rpc.SearchBy(word, by)
if err == nil {
usedIndex = i
break
}
}
if err != nil {
return nil, err
}
if len(pkgS) == 1 {
if sortS {
2017-08-04 09:26:53 +00:00
sort.Sort(aurQuery(r))
}
return r, err
}
var aq aurQuery
var n int
for i := range r {
match := true
2020-06-15 03:58:22 +00:00
for j, pkgN := range pkgS {
if usedIndex == j {
continue
}
if !(strings.Contains(r[i].Name, pkgN) || strings.Contains(strings.ToLower(r[i].Description), pkgN)) {
match = false
break
}
}
if match {
n++
aq = append(aq, r[i])
}
}
if sortS {
sort.Sort(aq)
}
return aq, err
}
2017-04-29 17:12:12 +00:00
// SyncSearch presents a query to the local repos and to the AUR.
func syncSearch(pkgS []string) (err error) {
2018-06-04 19:36:10 +00:00
pkgS = removeInvalidTargets(pkgS)
var aurErr error
var repoErr error
var aq aurQuery
var pq repoQuery
if mode == modeAUR || mode == modeAny {
2018-06-04 19:36:10 +00:00
aq, aurErr = narrowSearch(pkgS, true)
}
if mode == modeRepo || mode == modeAny {
2018-07-31 00:56:41 +00:00
pq, repoErr = queryRepo(pkgS)
2018-06-04 19:36:10 +00:00
if repoErr != nil {
return err
}
2017-04-29 17:12:12 +00:00
}
switch config.SortMode {
case topDown:
if mode == modeRepo || mode == modeAny {
2018-06-04 19:36:10 +00:00
pq.printSearch()
}
if mode == modeAUR || mode == modeAny {
aq.printSearch(1)
2018-06-04 19:36:10 +00:00
}
case bottomUp:
if mode == modeAUR || mode == modeAny {
2018-06-04 19:36:10 +00:00
aq.printSearch(1)
}
if mode == modeRepo || mode == modeAny {
pq.printSearch()
}
default:
return errors.New(gotext.Get("invalid sort mode. Fix with yay -Y --bottomup --save"))
2017-04-29 17:12:12 +00:00
}
2018-04-03 05:49:41 +00:00
if aurErr != nil {
text.Errorln(gotext.Get("error during AUR search: %s", aurErr))
text.Warnln(gotext.Get("Showing repo packages only"))
2018-04-03 05:49:41 +00:00
}
2017-04-29 17:12:12 +00:00
return nil
}
// SyncInfo serves as a pacman -Si for repo packages and AUR packages.
func syncInfo(pkgS []string) error {
2018-03-22 18:11:00 +00:00
var info []*rpc.Pkg
missing := false
pkgS = removeInvalidTargets(pkgS)
aurS, repoS, err := packageSlices(pkgS)
2017-04-29 17:12:12 +00:00
if err != nil {
return err
2017-04-29 17:12:12 +00:00
}
if len(aurS) != 0 {
2019-02-04 16:56:02 +00:00
noDB := make([]string, 0, len(aurS))
for _, pkg := range aurS {
2019-02-04 16:56:02 +00:00
_, name := splitDBFromName(pkg)
noDB = append(noDB, name)
}
2019-02-04 16:56:02 +00:00
info, err = aurInfoPrint(noDB)
if err != nil {
missing = true
fmt.Fprintln(os.Stderr, err)
}
}
// Repo always goes first
2018-01-06 13:50:54 +00:00
if len(repoS) != 0 {
arguments := cmdArgs.copy()
arguments.clearTargets()
arguments.addTarget(repoS...)
err = show(passToPacman(arguments))
2018-01-06 13:50:54 +00:00
if err != nil {
return err
2018-01-06 13:50:54 +00:00
}
}
if len(aurS) != len(info) {
missing = true
}
if len(info) != 0 {
for _, pkg := range info {
2018-03-22 18:11:00 +00:00
PrintInfo(pkg)
}
2017-04-29 17:12:12 +00:00
}
if missing {
err = fmt.Errorf("")
}
return err
2017-04-29 17:12:12 +00:00
}
// Search handles repo searches. Creates a RepoSearch struct.
2018-07-31 00:56:41 +00:00
func queryRepo(pkgInputN []string) (s repoQuery, err error) {
2019-02-04 16:56:02 +00:00
dbList, err := alpmHandle.SyncDBs()
if err != nil {
return
}
_ = dbList.ForEach(func(db alpm.DB) error {
if len(pkgInputN) == 0 {
pkgs := db.PkgCache()
s = append(s, pkgs.Slice()...)
} else {
pkgs := db.Search(pkgInputN)
s = append(s, pkgs.Slice()...)
}
return nil
})
if config.SortMode == bottomUp {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
}
}
return
}
// PackageSlices separates an input slice into aur and repo slices
func packageSlices(toCheck []string) (aur, repo []string, err error) {
2019-02-04 16:56:02 +00:00
dbList, err := alpmHandle.SyncDBs()
if err != nil {
return nil, nil, err
}
New install algorithm I have replaced the old install and dependancy algorithms with a new design that attemps to be more pacaur like. Mostly in minimizing user input. Ask every thing first then do everything with no need for more user input. It is not yet fully complete but is finished enough so that it works, should not fail in most cases and provides a base for more contributors to help address the existing problems. The new install chain is as follows: Source info about the provided targets Fetch a list of all dependancies needed to install targets I put alot of effort into fetching the dependancy tree while making the least amount of aur requests as possible. I'm actually very happy with how it turned out and yay wil now resolve dependancies noticably faster than pacaur when there are many aur dependancies. Install repo targets by passing to pacman Print dependancy tree and ask to confirm Ask to clean build if directory already exists Download all pkgbuilds Ask to edit all pkgbuilds Ask to continue with the install Download the sources for each packagebuild Build and install every package using -s to get repo deps and -i to install Ask to remove make dependancies There are still a lot of things that need to be done for a fully working system. Here are the problems I found with this system, either new or existing: Formating I am not so good at formatting myself, I thought best to leave it until last so I could get feedback on how it should look and help implementing it. Dependancy tree The dependancy tree is usually correct although I have noticed times where it doesnt detect all the dependancies that it should. I have only noticed this when there are circular dependancies so i think this might be the cause. It's not a big deal currently because makepkg -i installed repo deps for us which handles the repo deps for us and will get the correct ones. So yay might not list all the dependancies. but they will get installed so I consider this a visual bug. I have yet to see any circular dependancies in the AUR so I can not say what will happend but I#m guessing that it will break. Versioned packages/dependencies Targets and dependancies with version constriants such as 'linux>=4.1' will not be checked on the aur side of things but will be checked on the repo side. Ignorepkg/Ignoregroup Currently I do not handle this in any way but it shouldn't be too hard to implement. Conflict checking This is not currently implemented either Split Paclages Split packages are not Handles properly. If we only specify one package so install from a split package makepkg -i ends up installing them all anyway. If we specify more than one (n) package it will actually build the package base n times and reinstall every split package n times. Makepkg To get things working I decided to keep using the makepkg -i method. I plan to eventually replace this with a pacman -U based method. This should allow passing args such as --dbpath and --config to aur packages aswell as help solve some problems such as the split packages. Clean build I plan to improve the clean build choice to be a little more smart and instead of check if the directory exists, check if the package is already build and if so skip the build all together.
2018-01-17 21:48:23 +00:00
for _, _pkg := range toCheck {
2019-02-04 16:56:02 +00:00
db, name := splitDBFromName(_pkg)
found := false
if db == "aur" || mode == modeAUR {
aur = append(aur, _pkg)
continue
} else if db != "" || mode == modeRepo {
repo = append(repo, _pkg)
continue
}
2019-02-04 16:56:02 +00:00
_ = dbList.ForEach(func(db alpm.DB) error {
2019-02-14 20:44:39 +00:00
if db.Pkg(name) != nil {
found = true
return fmt.Errorf("")
}
return nil
})
if !found {
2019-02-14 20:44:39 +00:00
found = !dbList.FindGroupPkgs(name).Empty()
New install algorithm I have replaced the old install and dependancy algorithms with a new design that attemps to be more pacaur like. Mostly in minimizing user input. Ask every thing first then do everything with no need for more user input. It is not yet fully complete but is finished enough so that it works, should not fail in most cases and provides a base for more contributors to help address the existing problems. The new install chain is as follows: Source info about the provided targets Fetch a list of all dependancies needed to install targets I put alot of effort into fetching the dependancy tree while making the least amount of aur requests as possible. I'm actually very happy with how it turned out and yay wil now resolve dependancies noticably faster than pacaur when there are many aur dependancies. Install repo targets by passing to pacman Print dependancy tree and ask to confirm Ask to clean build if directory already exists Download all pkgbuilds Ask to edit all pkgbuilds Ask to continue with the install Download the sources for each packagebuild Build and install every package using -s to get repo deps and -i to install Ask to remove make dependancies There are still a lot of things that need to be done for a fully working system. Here are the problems I found with this system, either new or existing: Formating I am not so good at formatting myself, I thought best to leave it until last so I could get feedback on how it should look and help implementing it. Dependancy tree The dependancy tree is usually correct although I have noticed times where it doesnt detect all the dependancies that it should. I have only noticed this when there are circular dependancies so i think this might be the cause. It's not a big deal currently because makepkg -i installed repo deps for us which handles the repo deps for us and will get the correct ones. So yay might not list all the dependancies. but they will get installed so I consider this a visual bug. I have yet to see any circular dependancies in the AUR so I can not say what will happend but I#m guessing that it will break. Versioned packages/dependencies Targets and dependancies with version constriants such as 'linux>=4.1' will not be checked on the aur side of things but will be checked on the repo side. Ignorepkg/Ignoregroup Currently I do not handle this in any way but it shouldn't be too hard to implement. Conflict checking This is not currently implemented either Split Paclages Split packages are not Handles properly. If we only specify one package so install from a split package makepkg -i ends up installing them all anyway. If we specify more than one (n) package it will actually build the package base n times and reinstall every split package n times. Makepkg To get things working I decided to keep using the makepkg -i method. I plan to eventually replace this with a pacman -U based method. This should allow passing args such as --dbpath and --config to aur packages aswell as help solve some problems such as the split packages. Clean build I plan to improve the clean build choice to be a little more smart and instead of check if the directory exists, check if the package is already build and if so skip the build all together.
2018-01-17 21:48:23 +00:00
}
if found {
repo = append(repo, _pkg)
New install algorithm I have replaced the old install and dependancy algorithms with a new design that attemps to be more pacaur like. Mostly in minimizing user input. Ask every thing first then do everything with no need for more user input. It is not yet fully complete but is finished enough so that it works, should not fail in most cases and provides a base for more contributors to help address the existing problems. The new install chain is as follows: Source info about the provided targets Fetch a list of all dependancies needed to install targets I put alot of effort into fetching the dependancy tree while making the least amount of aur requests as possible. I'm actually very happy with how it turned out and yay wil now resolve dependancies noticably faster than pacaur when there are many aur dependancies. Install repo targets by passing to pacman Print dependancy tree and ask to confirm Ask to clean build if directory already exists Download all pkgbuilds Ask to edit all pkgbuilds Ask to continue with the install Download the sources for each packagebuild Build and install every package using -s to get repo deps and -i to install Ask to remove make dependancies There are still a lot of things that need to be done for a fully working system. Here are the problems I found with this system, either new or existing: Formating I am not so good at formatting myself, I thought best to leave it until last so I could get feedback on how it should look and help implementing it. Dependancy tree The dependancy tree is usually correct although I have noticed times where it doesnt detect all the dependancies that it should. I have only noticed this when there are circular dependancies so i think this might be the cause. It's not a big deal currently because makepkg -i installed repo deps for us which handles the repo deps for us and will get the correct ones. So yay might not list all the dependancies. but they will get installed so I consider this a visual bug. I have yet to see any circular dependancies in the AUR so I can not say what will happend but I#m guessing that it will break. Versioned packages/dependencies Targets and dependancies with version constriants such as 'linux>=4.1' will not be checked on the aur side of things but will be checked on the repo side. Ignorepkg/Ignoregroup Currently I do not handle this in any way but it shouldn't be too hard to implement. Conflict checking This is not currently implemented either Split Paclages Split packages are not Handles properly. If we only specify one package so install from a split package makepkg -i ends up installing them all anyway. If we specify more than one (n) package it will actually build the package base n times and reinstall every split package n times. Makepkg To get things working I decided to keep using the makepkg -i method. I plan to eventually replace this with a pacman -U based method. This should allow passing args such as --dbpath and --config to aur packages aswell as help solve some problems such as the split packages. Clean build I plan to improve the clean build choice to be a little more smart and instead of check if the directory exists, check if the package is already build and if so skip the build all together.
2018-01-17 21:48:23 +00:00
} else {
aur = append(aur, _pkg)
}
}
return aur, repo, nil
}
// HangingPackages returns a list of packages installed as deps
// and unneeded by the system
2018-03-23 23:45:46 +00:00
// removeOptional decides whether optional dependencies are counted or not
func hangingPackages(removeOptional bool) (hanging []string, err error) {
2019-02-04 16:56:02 +00:00
localDB, err := alpmHandle.LocalDB()
if err != nil {
return
}
2018-03-23 23:45:46 +00:00
// safePackages represents every package in the system in one of 3 states
// State = 0 - Remove package from the system
// State = 1 - Keep package in the system; need to iterate over dependencies
// State = 2 - Keep package and have iterated over dependencies
safePackages := make(map[string]uint8)
// provides stores a mapping from the provides name back to the original package name
provides := make(stringset.MapStringSet)
2019-02-04 16:56:02 +00:00
packages := localDB.PkgCache()
2018-03-23 23:45:46 +00:00
// Mark explicit dependencies and enumerate the provides list
setupResources := func(pkg alpm.Package) error {
if pkg.Reason() == alpm.PkgReasonExplicit {
safePackages[pkg.Name()] = 1
} else {
safePackages[pkg.Name()] = 0
}
_ = pkg.Provides().ForEach(func(dep alpm.Depend) error {
provides.Add(dep.Name, pkg.Name())
2018-03-23 23:45:46 +00:00
return nil
})
return nil
}
_ = packages.ForEach(setupResources)
2018-03-23 23:45:46 +00:00
iterateAgain := true
processDependencies := func(pkg alpm.Package) error {
if state := safePackages[pkg.Name()]; state == 0 || state == 2 {
return nil
}
2018-03-23 23:45:46 +00:00
safePackages[pkg.Name()] = 2
// Update state for dependencies
markDependencies := func(dep alpm.Depend) error {
// Don't assume a dependency is installed
state, ok := safePackages[dep.Name]
if !ok {
// Check if dep is a provides rather than actual package name
2018-03-24 12:37:31 +00:00
if pset, ok2 := provides[dep.Name]; ok2 {
for p := range pset {
if safePackages[p] == 0 {
iterateAgain = true
safePackages[p] = 1
}
}
2018-03-23 23:45:46 +00:00
}
return nil
}
if state == 0 {
iterateAgain = true
safePackages[dep.Name] = 1
}
return nil
}
_ = pkg.Depends().ForEach(markDependencies)
2018-03-23 23:45:46 +00:00
if !removeOptional {
_ = pkg.OptionalDepends().ForEach(markDependencies)
}
return nil
}
2018-03-23 23:45:46 +00:00
for iterateAgain {
iterateAgain = false
_ = packages.ForEach(processDependencies)
2018-03-23 23:45:46 +00:00
}
// Build list of packages to be removed
_ = packages.ForEach(func(pkg alpm.Package) error {
2018-03-23 23:45:46 +00:00
if safePackages[pkg.Name()] == 0 {
hanging = append(hanging, pkg.Name())
}
return nil
})
return hanging, err
}
func lastBuildTime() (time.Time, error) {
var lastTime time.Time
pkgs, _, _, _, err := filterPackages()
if err != nil {
return lastTime, err
}
for _, pkg := range pkgs {
thisTime := pkg.BuildDate()
if thisTime.After(lastTime) {
lastTime = thisTime
}
}
return lastTime, nil
}
// Statistics returns statistics about packages installed in system
func statistics() (*struct {
Totaln int
Expln int
TotalSize int64
}, error) {
var tS int64 // TotalSize
var nPkg int
var ePkg int
2019-02-04 16:56:02 +00:00
localDB, err := alpmHandle.LocalDB()
if err != nil {
return nil, err
}
2019-02-04 16:56:02 +00:00
for _, pkg := range localDB.PkgCache().Slice() {
tS += pkg.ISize()
nPkg++
if pkg.Reason() == 0 {
ePkg++
}
}
info := &struct {
Totaln int
Expln int
TotalSize int64
}{
nPkg, ePkg, tS,
}
return info, err
}
// Queries the aur for information about specified packages.
// All packages should be queried in a single rpc request except when the number
// of packages exceeds the number set in config.RequestSplitN.
// If the number does exceed config.RequestSplitN multiple rpc requests will be
// performed concurrently.
func aurInfo(names []string, warnings *aurWarnings) ([]*rpc.Pkg, error) {
2018-03-22 18:11:00 +00:00
info := make([]*rpc.Pkg, 0, len(names))
seen := make(map[string]int)
2018-02-19 04:25:36 +00:00
var mux sync.Mutex
var wg sync.WaitGroup
var errs multierror.MultiError
2018-02-19 04:25:36 +00:00
makeRequest := func(n, max int) {
defer wg.Done()
2018-02-19 04:25:36 +00:00
tempInfo, requestErr := rpc.Info(names[n:max])
2018-08-02 16:14:57 +00:00
errs.Add(requestErr)
2018-02-19 04:25:36 +00:00
if requestErr != nil {
return
}
mux.Lock()
for i := range tempInfo {
info = append(info, &tempInfo[i])
2018-03-22 18:11:00 +00:00
}
2018-02-19 04:25:36 +00:00
mux.Unlock()
}
for n := 0; n < len(names); n += config.RequestSplitN {
max := intrange.Min(len(names), n+config.RequestSplitN)
2018-02-19 04:25:36 +00:00
wg.Add(1)
go makeRequest(n, max)
}
wg.Wait()
2018-08-02 16:14:57 +00:00
if err := errs.Return(); err != nil {
2018-02-19 04:25:36 +00:00
return info, err
}
for k, pkg := range info {
seen[pkg.Name] = k
}
for _, name := range names {
i, ok := seen[name]
2019-11-11 07:15:04 +00:00
if !ok && !warnings.Ignore.Get(name) {
warnings.Missing = append(warnings.Missing, name)
continue
}
pkg := info[i]
2019-11-11 07:15:04 +00:00
if pkg.Maintainer == "" && !warnings.Ignore.Get(name) {
warnings.Orphans = append(warnings.Orphans, name)
}
2019-11-11 07:15:04 +00:00
if pkg.OutOfDate != 0 && !warnings.Ignore.Get(name) {
warnings.OutOfDate = append(warnings.OutOfDate, name)
}
}
return info, nil
}
func aurInfoPrint(names []string) ([]*rpc.Pkg, error) {
text.OperationInfoln(gotext.Get("Querying AUR..."))
warnings := &aurWarnings{}
info, err := aurInfo(names, warnings)
if err != nil {
return info, err
}
2018-05-20 15:17:05 +00:00
warnings.print()
return info, nil
}