Implemmented github VCS updates. Getting hammered by rate limiting

aur/aur.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"strings"
 
+	vcs "github.com/jguer/yay/aur/vcs"
 	"github.com/jguer/yay/config"
 	"github.com/jguer/yay/pacman"
 	rpc "github.com/mikkeloscar/aur"
@@ -16,6 +17,8 @@ import (
 // BaseURL givers the AUR default address.
 const BaseURL string = "https://aur.archlinux.org"
 
+var specialDBsauce bool = false
+
 // NarrowSearch searches AUR and narrows based on subarguments
 func NarrowSearch(pkgS []string, sortS bool) (Query, error) {
 	if len(pkgS) == 0 {
@@ -67,7 +70,7 @@ func Install(pkgName []string, flags []string) (err error) {
 	}
 
 	if len(q) != len(pkgName) {
-		return fmt.Errorf("Some package from list\n%+v\ndoes not exist", pkgName)
+		fmt.Printf("Some package from list\n%+v\ndoes not exist", pkgName)
 	}
 
 	var finalrm []string
@@ -86,10 +89,49 @@ func Install(pkgName []string, flags []string) (err error) {
 	return err
 }
 
+// CreateDevelDB forces yay to create a DB of the existing development packages
+func CreateDevelDB() error {
+	foreign, err := pacman.ForeignPackages()
+	if err != nil {
+		return err
+	}
+
+	keys := make([]string, len(foreign))
+	i := 0
+	for k := range foreign {
+		keys[i] = k
+		i++
+	}
+
+	config.YayConf.NoConfirm = true
+	specialDBsauce = true
+	err = Install(keys, nil)
+	return err
+}
+
+func DevelUpgrade(flags []string) {
+
+}
 // Upgrade tries to update every foreign package installed in the system
 func Upgrade(flags []string) error {
 	fmt.Println("\x1b[1;36;1m::\x1b[0m\x1b[1m Starting AUR upgrade...\x1b[0m")
 
+	if config.YayConf.Devel {
+		fmt.Println(" Checking development packages...")
+		develUpdates := vcs.CheckUpdates()
+		if len(develUpdates) != 0 {
+			// Install updated packages
+			if !config.ContinueTask("\nProceed with upgrade?", "nN") {
+				return nil
+			}
+
+			err := Install(develUpdates, flags)
+			if err != nil {
+				fmt.Println(err)
+			}
+		}
+	}
+
 	foreign, err := pacman.ForeignPackages()
 	if err != nil {
 		return err

aur/result.go

@@ -5,9 +5,11 @@ import (
 	"os"
 	"os/exec"
 
+	vcs "github.com/jguer/yay/aur/vcs"
 	"github.com/jguer/yay/config"
 	"github.com/jguer/yay/pacman"
 	rpc "github.com/mikkeloscar/aur"
+	gopkg "github.com/mikkeloscar/gopkgbuild"
 )
 
 // PkgDependencies returns package dependencies not installed belonging to AUR
@@ -88,6 +90,24 @@ func PkgInstall(a *rpc.Pkg, flags []string) (finalmdeps []string, err error) {
 		editcmd.Run()
 	}
 
+	pkgb, err := gopkg.ParseSRCINFO(dir + ".SRCINFO")
+	if err == nil {
+		for _, pkgsource := range pkgb.Source {
+			owner, repo := vcs.ParseSource(pkgsource)
+			if owner != "" && repo != "" {
+				err = vcs.BranchInfo(a.Name, owner, repo)
+				if err != nil {
+					fmt.Println(err)
+				}
+				vcs.SaveBranchInfo()
+			}
+		}
+	}
+
+	if specialDBsauce {
+		return
+	}
+
 	runDeps, makeDeps, err := PkgDependencies(a)
 	if err != nil {
 		return

aur/vcs/github.go

@@ -10,9 +10,11 @@ import (
 
 // branch contains the information of a repository branch
 type branch struct {
-	Name string `json:"name"`
-	SHA  string `json:"sha"`
-	URL  string `json:"url"`
+	Name   string `json:"name"`
+	Commit struct {
+		SHA string `json:"sha"`
+		URL string `json:"url"`
+	} `json:"commit"`
 }
 
 type branches []branch
@@ -29,11 +31,16 @@ type infos []Info
 var savedInfo infos
 var configfile string
 
+// Updated returns if database has been updated
+var Updated bool
+
 func init() {
+	Updated = false
 	configfile = os.Getenv("HOME") + "/.config/yay/yay_vcs.json"
 
 	if _, err := os.Stat(configfile); os.IsNotExist(err) {
 		_ = os.MkdirAll(os.Getenv("HOME")+"/.config/yay", 0755)
+		return
 	}
 
 	file, err := os.Open(configfile)
@@ -50,6 +57,11 @@ func init() {
 
 // ParseSource returns owner and repo from source
 func ParseSource(source string) (owner string, repo string) {
+	if !(strings.Contains(source, "git://") ||
+		strings.Contains(source, ".git") ||
+		strings.Contains(source, "git+https://")) {
+		return
+	}
 	split := strings.Split(source, "github.com/")
 	if len(split) > 1 {
 		secondSplit := strings.Split(split[1], "/")
@@ -81,7 +93,7 @@ func (info *Info) needsUpdate() bool {
 
 	for _, e := range newRepo {
 		if e.Name == "master" {
-			if e.SHA != info.SHA {
+			if e.Commit.SHA != info.SHA {
 				return true
 			} else {
 				return false
@@ -100,9 +112,9 @@ func CheckUpdates() (toUpdate []string) {
 	return
 }
 
-func inStore(pkgname string) *Info {
+func inStore(url string) *Info {
 	for i, e := range savedInfo {
-		if pkgname == e.Package {
+		if url == e.URL {
 			return &savedInfo[i]
 		}
 	}
@@ -111,6 +123,7 @@ func inStore(pkgname string) *Info {
 
 // BranchInfo updates saved information
 func BranchInfo(pkgname string, owner string, repo string) (err error) {
+	Updated = true
 	var newRepo branches
 	url := "https://api.github.com/repos/" + owner + "/" + repo + "/branches"
 	r, err := http.Get(url)
@@ -121,16 +134,16 @@ func BranchInfo(pkgname string, owner string, repo string) (err error) {
 
 	_ = json.NewDecoder(r.Body).Decode(&newRepo)
 
-	packinfo := inStore(pkgname)
+	packinfo := inStore(url)
 
 	for _, e := range newRepo {
 		if e.Name == "master" {
 			if packinfo != nil {
 				packinfo.Package = pkgname
 				packinfo.URL = url
-				packinfo.SHA = e.SHA
+				packinfo.SHA = e.Commit.SHA
 			} else {
-				savedInfo = append(savedInfo, Info{Package: pkgname, URL: url, SHA: e.SHA})
+				savedInfo = append(savedInfo, Info{Package: pkgname, URL: url, SHA: e.Commit.SHA})
 			}
 		}
 	}

aur/vcs/github_test.go

@@ -12,20 +12,20 @@ func TestParsing(t *testing.T) {
 	}
 
 	neovim := source{sourceurl: "git+https://github.com/neovim/neovim.git"}
-	neovim.owner, neovim.repo = parseSource(neovim.sourceurl)
+	neovim.owner, neovim.repo = ParseSource(neovim.sourceurl)
 
 	if neovim.owner != "neovim" || neovim.repo != "neovim" {
 		t.Fatalf("Expected to find neovim/neovim, found %+v/%+v", neovim.owner, neovim.repo)
 	}
 
 	yay := source{sourceurl: "git://github.com/jguer/yay.git#branch=master"}
-	yay.owner, yay.repo = parseSource(yay.sourceurl)
+	yay.owner, yay.repo = ParseSource(yay.sourceurl)
 	if yay.owner != "jguer" || yay.repo != "yay" {
 		t.Fatalf("Expected to find jguer/yay, found %+v/%+v", yay.owner, yay.repo)
 	}
 
 	ack := source{sourceurl: "git://github.com/davidgiven/ack"}
-	ack.owner, ack.repo = parseSource(ack.sourceurl)
+	ack.owner, ack.repo = ParseSource(ack.sourceurl)
 	if ack.owner != "davidgiven" || ack.repo != "ack" {
 		t.Fatalf("Expected to find davidgiven/ack, found %+v/%+v", ack.owner, ack.repo)
 	}

config/config.go

@@ -32,6 +32,7 @@ type Configuration struct {
 	MakepkgBin string `json:"makepkgbin"`
 	Shell      string `json:"-"`
 	NoConfirm  bool   `json:"noconfirm"`
+	Devel      bool   `json:"devel"`
 	PacmanBin  string `json:"pacmanbin"`
 	PacmanConf string `json:"pacmanconf"`
 	SearchMode int    `json:"-"`
@@ -115,6 +116,7 @@ func SaveConfig() error {
 func defaultSettings(config *Configuration) {
 	config.BuildDir = "/tmp/yaytmp/"
 	config.Editor = ""
+	config.Devel = false
 	config.MakepkgBin = "/usr/bin/makepkg"
 	config.NoConfirm = false
 	config.PacmanBin = "/usr/bin/pacman"

yay.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/jguer/yay/aur"
+	vcs "github.com/jguer/yay/aur/vcs"
 	"github.com/jguer/yay/config"
 	pac "github.com/jguer/yay/pacman"
 )
@@ -57,13 +58,18 @@ func parser() (op string, options []string, packages []string, changedConfig boo
 		}
 
 		if arg[0] == '-' && arg[1] == '-' {
+			changedConfig = true
 			switch arg {
-			case "--bottomup":
-				config.YayConf.SortMode = config.BottomUp
-				changedConfig = true
+			case "--gendb":
+				aur.CreateDevelDB()
+				vcs.SaveBranchInfo()
+				os.Exit(0)
+			case "--devel":
+				config.YayConf.Devel = true
+			case "--nodevel":
+				config.YayConf.Devel = false
 			case "--topdown":
 				config.YayConf.SortMode = config.TopDown
-				changedConfig = true
 			case "--complete":
 				config.YayConf.Shell = "sh"
 				complete()
@@ -133,6 +139,11 @@ func main() {
 	default:
 		err = config.PassToPacman(op, pkgs, options)
 	}
+
+	if vcs.Updated {
+		vcs.SaveBranchInfo()
+	}
+
 	if changedConfig {
 		config.SaveConfig()
 	}
@@ -207,12 +218,12 @@ func numberMenu(pkgS []string, flags []string) (err error) {
 	}
 
 	if len(repoInstall) != 0 {
-		config.PassToPacman("-S", repoInstall, flags)
+		err = config.PassToPacman("-S", repoInstall, flags)
 	}
 
 	if len(aurInstall) != 0 {
-		aur.Install(aurInstall, flags)
+		err = aur.Install(aurInstall, flags)
 	}
 
-	return nil
+	return err
 }