yay/vcs.go
morganamilo 57a8048cb8
Use goroutinuies for devel updates
The update process was already partly parallel: repo, aur and devel
updates where each given their own goroutine.

This gives two further layers of parallelization. Firstly the
`needsUpdate()` function is now run in parallel for each package checked.
One package may have many vcs sources so `needsUpdate()` also checks all
of its sources in parallel.

This gives an aproxamte 3x speedup for `yay -Su --devel` timing from
when the command starts to when the number menu apears.

unfortunately git://bitbucket.org/volumesoffun/polyvox.git never
resolves on my machine for some reason so it always hits the 5 second
timout period.

It then moves on to http:/bitbucket.org/volumesoffun/polyvox.git/ which
does resolve as expected. I have not looked into it but I fear this
applies to all gitbucket repos. Luckly they are few and far between.
2018-03-13 17:18:46 +00:00

188 lines
3.6 KiB
Go

package main
import (
"bytes"
"encoding/json"
"os"
"os/exec"
"strings"
"sync"
"time"
)
// Info contains the last commit sha of a repo
type vcsInfo map[string]shaInfos
type shaInfos map[string]shaInfo
type shaInfo struct {
Protocols []string `json:"protocols"`
Brach string `json:"branch"`
SHA string `json:"sha"`
}
// createDevelDB forces yay to create a DB of the existing development packages
func createDevelDB() error {
_, _, _, remoteNames, err := filterPackages()
if err != nil {
return err
}
config.NoConfirm = true
arguments := makeArguments()
arguments.addArg("gendb")
arguments.addTarget(remoteNames...)
err = install(arguments)
return err
}
// parseSource returns the git url, default branch and protocols it supports
func parseSource(source string) (url string, branch string, protocols []string) {
if !(strings.Contains(source, "git://") ||
strings.Contains(source, ".git") ||
strings.Contains(source, "git+https://")) {
return "", "", nil
}
split := strings.Split(source, "::")
source = split[len(split)-1]
split = strings.SplitN(source, "://", 2)
if len(split) != 2 {
return "", "", nil
}
protocols = strings.Split(split[0], "+")
split = strings.SplitN(split[1], "#", 2)
if len(split) == 2 {
secondSplit := strings.SplitN(split[1], "=", 2)
if secondSplit[0] != "branch" {
//source has #commit= or #tag= which makes them not vcs
//packages because they reference a specific point
return "", "", nil
}
if len(secondSplit) == 2 {
url = split[0]
branch = secondSplit[1]
}
} else {
url = split[0]
branch = "HEAD"
}
return
}
func updateVCSData(pkgName string, sources []string) {
if savedInfo == nil {
savedInfo = make(vcsInfo)
}
info := make(shaInfos)
for _, source := range sources {
url, branch, protocols := parseSource(source)
if url == "" || branch == "" {
continue
}
commit := getCommit(url, branch, protocols)
if commit == "" {
continue
}
info[url] = shaInfo{
protocols,
branch,
commit,
}
savedInfo[pkgName] = info
saveVCSInfo()
}
}
func getCommit(url string, branch string, protocols []string) string {
for _, protocol := range protocols {
var outbuf bytes.Buffer
cmd := exec.Command(config.GitBin, "ls-remote", protocol+"://"+url, branch)
cmd.Stdout = &outbuf
err := cmd.Start()
if err != nil {
continue
}
//for some reason
//git://bitbucket.org/volumesoffun/polyvox.git` hangs on my
//machine but using http:// instead of git does not hang.
//Introduce a time out so this can not hang
timer := time.AfterFunc(5*time.Second, func() {
cmd.Process.Kill()
})
err = cmd.Wait()
timer.Stop()
if err != nil {
continue
}
err = cmd.Run()
stdout := outbuf.String()
split := strings.Fields(stdout)
if len(split) < 2 {
continue
}
commit := split[0]
return commit
}
return ""
}
func (infos shaInfos) needsUpdate() bool {
var wg sync.WaitGroup
hasUpdate := false
checkHash := func(url string, info shaInfo) {
defer wg.Done()
hash := getCommit(url, info.Brach, info.Protocols)
if hash != "" && hash != info.SHA {
hasUpdate = true
}
}
for url, info := range infos {
wg.Add(1)
go checkHash(url, info)
}
wg.Wait()
return hasUpdate
}
func inStore(pkgName string) shaInfos {
return savedInfo[pkgName]
}
func saveVCSInfo() error {
marshalledinfo, err := json.MarshalIndent(savedInfo, "", "\t")
if err != nil || string(marshalledinfo) == "null" {
return err
}
in, err := os.OpenFile(vcsFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer in.Close()
_, err = in.Write(marshalledinfo)
if err != nil {
return err
}
err = in.Sync()
return err
}