feat(aur): add option to limit concurrent downloads (#1768)

* feat(aur): add option to limit concurrent downloads Adds to config file option 'maxconcurrentdownloads' which if set to value != 0, limits number of concurrent AUR downloads to specified. Fixes #1763. * fix lint issue
2024-07-22 10:48:05 +00:00 · 2022-06-28 12:05:29 +02:00 · 2022-06-28 12:05:29 +02:00 · e0006ec272
parent 18fbc24e68
commit e0006ec272
4 changed files with 113 additions and 100 deletions
--- a/aur_source.go
+++ b/aur_source.go
@ -65,7 +65,7 @@ func downloadPKGBUILDSourceWorker(ctx context.Context, wg *sync.WaitGroup, dest
 }

 func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilder, dest string,
-	bases []dep.Base, incompatible stringset.StringSet) error {
+	bases []dep.Base, incompatible stringset.StringSet, maxConcurrentDownloads int) error {
 	if len(bases) == 1 {
 		return downloadPKGBUILDSource(ctx, cmdBuilder, dest, bases[0].Pkgbase(), incompatible)
 	}
@ -78,6 +78,10 @@ func downloadPKGBUILDSourceFanout(ctx context.Context, cmdBuilder exe.ICmdBuilde
 		fanInChanErrors = make(chan error)
 	)

+	if maxConcurrentDownloads != 0 {
+		numOfWorkers = maxConcurrentDownloads
+	}
+
 	go func() {
 		for _, base := range bases {
 			c <- base.Pkgbase()
--- a/aur_source_test.go
+++ b/aur_source_test.go
@ -2,6 +2,7 @@ package main

 import (
 	"context"
+	"fmt"
 	"os/exec"
 	"sync/atomic"
 	"testing"
@ -82,13 +83,6 @@ func Test_downloadPKGBUILDSourceError(t *testing.T) {
 // THEN 5 calls should be made to makepkg
 func Test_downloadPKGBUILDSourceFanout(t *testing.T) {
 	t.Parallel()
-	cmdBuilder := &TestMakepkgBuilder{
-		parentBuilder: &exe.CmdBuilder{
-			MakepkgConfPath: "/etc/not.conf",
-			MakepkgFlags:    []string{"--nocheck"}, MakepkgBin: "makepkg",
-		},
-		test: t,
-	}

 	bases := []dep.Base{
 		{&aur.Pkg{PackageBase: "yay"}},
@ -98,9 +92,21 @@ func Test_downloadPKGBUILDSourceFanout(t *testing.T) {
 		{&aur.Pkg{PackageBase: "yay-v12"}},
 	}

-	err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
-	assert.NoError(t, err)
-	assert.Equal(t, 5, int(cmdBuilder.passes))
+	for _, maxConcurrentDownloads := range []int{0, 3} {
+		t.Run(fmt.Sprintf("maxconcurrentdownloads set to %d", maxConcurrentDownloads), func(t *testing.T) {
+			cmdBuilder := &TestMakepkgBuilder{
+				parentBuilder: &exe.CmdBuilder{
+					MakepkgConfPath: "/etc/not.conf",
+					MakepkgFlags:    []string{"--nocheck"}, MakepkgBin: "makepkg",
+				},
+				test: t,
+			}
+
+			err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), maxConcurrentDownloads)
+			assert.NoError(t, err)
+			assert.Equal(t, 5, int(cmdBuilder.passes))
+		})
+	}
 }

 // GIVEN 1 package
@ -120,7 +126,7 @@ func Test_downloadPKGBUILDSourceFanoutNoCC(t *testing.T) {
 		{&aur.Pkg{PackageBase: "yay"}},
 	}

-	err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
+	err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0)
 	assert.NoError(t, err)
 	assert.Equal(t, 1, int(cmdBuilder.passes))
 }
@ -147,7 +153,7 @@ func Test_downloadPKGBUILDSourceFanoutError(t *testing.T) {
 		{&aur.Pkg{PackageBase: "yay-v12"}},
 	}

-	err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make())
+	err := downloadPKGBUILDSourceFanout(context.TODO(), cmdBuilder, "/tmp", bases, stringset.Make(), 0)
 	assert.Error(t, err)
 	assert.Equal(t, 5, int(cmdBuilder.passes))
 	assert.Len(t, err.(*multierror.MultiError).Errors, 5)
--- a/install.go
+++ b/install.go
@ -330,7 +330,8 @@ func install(ctx context.Context, cmdArgs *parser.Arguments, dbExecutor db.Execu
 			config.AURURL, config.Runtime.CompletionPath, config.CompletionInterval, false)
 	}()

-	if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir, do.Aur, incompatible); errP != nil {
+	if errP := downloadPKGBUILDSourceFanout(ctx, config.Runtime.CmdBuilder, config.BuildDir,
+		do.Aur, incompatible, config.MaxConcurrentDownloads); errP != nil {
 		text.Errorln(errP)
 	}

--- a/pkg/settings/config.go
+++ b/pkg/settings/config.go
@ -30,50 +30,51 @@ var NoConfirm = false

 // Configuration stores yay's config.
 type Configuration struct {
-	AURURL             string   `json:"aururl"`
-	BuildDir           string   `json:"buildDir"`
-	Editor             string   `json:"editor"`
-	EditorFlags        string   `json:"editorflags"`
-	MakepkgBin         string   `json:"makepkgbin"`
-	MakepkgConf        string   `json:"makepkgconf"`
-	PacmanBin          string   `json:"pacmanbin"`
-	PacmanConf         string   `json:"pacmanconf"`
-	ReDownload         string   `json:"redownload"`
-	ReBuild            string   `json:"rebuild"`
-	AnswerClean        string   `json:"answerclean"`
-	AnswerDiff         string   `json:"answerdiff"`
-	AnswerEdit         string   `json:"answeredit"`
-	AnswerUpgrade      string   `json:"answerupgrade"`
-	GitBin             string   `json:"gitbin"`
-	GpgBin             string   `json:"gpgbin"`
-	GpgFlags           string   `json:"gpgflags"`
-	MFlags             string   `json:"mflags"`
-	SortBy             string   `json:"sortby"`
-	SearchBy           string   `json:"searchby"`
-	GitFlags           string   `json:"gitflags"`
-	RemoveMake         string   `json:"removemake"`
-	SudoBin            string   `json:"sudobin"`
-	SudoFlags          string   `json:"sudoflags"`
-	RequestSplitN      int      `json:"requestsplitn"`
-	CompletionInterval int      `json:"completionrefreshtime"`
-	BottomUp           bool     `json:"bottomup"`
-	SudoLoop           bool     `json:"sudoloop"`
-	TimeUpdate         bool     `json:"timeupdate"`
-	Devel              bool     `json:"devel"`
-	CleanAfter         bool     `json:"cleanAfter"`
-	Provides           bool     `json:"provides"`
-	PGPFetch           bool     `json:"pgpfetch"`
-	UpgradeMenu        bool     `json:"upgrademenu"`
-	CleanMenu          bool     `json:"cleanmenu"`
-	DiffMenu           bool     `json:"diffmenu"`
-	EditMenu           bool     `json:"editmenu"`
-	CombinedUpgrade    bool     `json:"combinedupgrade"`
-	UseAsk             bool     `json:"useask"`
-	BatchInstall       bool     `json:"batchinstall"`
-	SingleLineResults  bool     `json:"singlelineresults"`
-	SeparateSources    bool     `json:"separatesources"`
-	Runtime            *Runtime `json:"-"`
-	Version            string   `json:"version"`
+	AURURL                 string   `json:"aururl"`
+	BuildDir               string   `json:"buildDir"`
+	Editor                 string   `json:"editor"`
+	EditorFlags            string   `json:"editorflags"`
+	MakepkgBin             string   `json:"makepkgbin"`
+	MakepkgConf            string   `json:"makepkgconf"`
+	PacmanBin              string   `json:"pacmanbin"`
+	PacmanConf             string   `json:"pacmanconf"`
+	ReDownload             string   `json:"redownload"`
+	ReBuild                string   `json:"rebuild"`
+	AnswerClean            string   `json:"answerclean"`
+	AnswerDiff             string   `json:"answerdiff"`
+	AnswerEdit             string   `json:"answeredit"`
+	AnswerUpgrade          string   `json:"answerupgrade"`
+	GitBin                 string   `json:"gitbin"`
+	GpgBin                 string   `json:"gpgbin"`
+	GpgFlags               string   `json:"gpgflags"`
+	MFlags                 string   `json:"mflags"`
+	SortBy                 string   `json:"sortby"`
+	SearchBy               string   `json:"searchby"`
+	GitFlags               string   `json:"gitflags"`
+	RemoveMake             string   `json:"removemake"`
+	SudoBin                string   `json:"sudobin"`
+	SudoFlags              string   `json:"sudoflags"`
+	RequestSplitN          int      `json:"requestsplitn"`
+	CompletionInterval     int      `json:"completionrefreshtime"`
+	MaxConcurrentDownloads int      `json:"maxconcurrentdownloads"`
+	BottomUp               bool     `json:"bottomup"`
+	SudoLoop               bool     `json:"sudoloop"`
+	TimeUpdate             bool     `json:"timeupdate"`
+	Devel                  bool     `json:"devel"`
+	CleanAfter             bool     `json:"cleanAfter"`
+	Provides               bool     `json:"provides"`
+	PGPFetch               bool     `json:"pgpfetch"`
+	UpgradeMenu            bool     `json:"upgrademenu"`
+	CleanMenu              bool     `json:"cleanmenu"`
+	DiffMenu               bool     `json:"diffmenu"`
+	EditMenu               bool     `json:"editmenu"`
+	CombinedUpgrade        bool     `json:"combinedupgrade"`
+	UseAsk                 bool     `json:"useask"`
+	BatchInstall           bool     `json:"batchinstall"`
+	SingleLineResults      bool     `json:"singlelineresults"`
+	SeparateSources        bool     `json:"separatesources"`
+	Runtime                *Runtime `json:"-"`
+	Version                string   `json:"version"`
 }

 // SaveConfig writes yay config to file.
@ -178,48 +179,49 @@ func (c *Configuration) setPrivilegeElevator() error {

 func DefaultConfig(version string) *Configuration {
 	return &Configuration{
-		AURURL:             "https://aur.archlinux.org",
-		BuildDir:           os.ExpandEnv("$HOME/.cache/yay"),
-		CleanAfter:         false,
-		Editor:             "",
-		EditorFlags:        "",
-		Devel:              false,
-		MakepkgBin:         "makepkg",
-		MakepkgConf:        "",
-		PacmanBin:          "pacman",
-		PGPFetch:           true,
-		PacmanConf:         "/etc/pacman.conf",
-		GpgFlags:           "",
-		MFlags:             "",
-		GitFlags:           "",
-		BottomUp:           true,
-		CompletionInterval: 7,
-		SortBy:             "votes",
-		SearchBy:           "name-desc",
-		SudoLoop:           false,
-		GitBin:             "git",
-		GpgBin:             "gpg",
-		SudoBin:            "sudo",
-		SudoFlags:          "",
-		TimeUpdate:         false,
-		RequestSplitN:      150,
-		ReDownload:         "no",
-		ReBuild:            "no",
-		BatchInstall:       false,
-		AnswerClean:        "",
-		AnswerDiff:         "",
-		AnswerEdit:         "",
-		AnswerUpgrade:      "",
-		RemoveMake:         "ask",
-		Provides:           true,
-		UpgradeMenu:        true,
-		CleanMenu:          true,
-		DiffMenu:           true,
-		EditMenu:           false,
-		UseAsk:             false,
-		CombinedUpgrade:    false,
-		SeparateSources:    true,
-		Version:            version,
+		AURURL:                 "https://aur.archlinux.org",
+		BuildDir:               os.ExpandEnv("$HOME/.cache/yay"),
+		CleanAfter:             false,
+		Editor:                 "",
+		EditorFlags:            "",
+		Devel:                  false,
+		MakepkgBin:             "makepkg",
+		MakepkgConf:            "",
+		PacmanBin:              "pacman",
+		PGPFetch:               true,
+		PacmanConf:             "/etc/pacman.conf",
+		GpgFlags:               "",
+		MFlags:                 "",
+		GitFlags:               "",
+		BottomUp:               true,
+		CompletionInterval:     7,
+		MaxConcurrentDownloads: 0,
+		SortBy:                 "votes",
+		SearchBy:               "name-desc",
+		SudoLoop:               false,
+		GitBin:                 "git",
+		GpgBin:                 "gpg",
+		SudoBin:                "sudo",
+		SudoFlags:              "",
+		TimeUpdate:             false,
+		RequestSplitN:          150,
+		ReDownload:             "no",
+		ReBuild:                "no",
+		BatchInstall:           false,
+		AnswerClean:            "",
+		AnswerDiff:             "",
+		AnswerEdit:             "",
+		AnswerUpgrade:          "",
+		RemoveMake:             "ask",
+		Provides:               true,
+		UpgradeMenu:            true,
+		CleanMenu:              true,
+		DiffMenu:               true,
+		EditMenu:               false,
+		UseAsk:                 false,
+		CombinedUpgrade:        false,
+		SeparateSources:        true,
+		Version:                version,
 	}
 }