Merge pull request #2360 from vrothberg/parallel-search

podman-search: run in parallel
This commit is contained in:
OpenShift Merge Robot 2019-02-20 18:44:40 +01:00 committed by GitHub
commit 148d46766f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 352 additions and 265 deletions

13
API.md
View file

@ -107,7 +107,7 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
[func RestartPod(name: string) string](#RestartPod) [func RestartPod(name: string) string](#RestartPod)
[func SearchImages(query: string, limit: , tlsVerify: ) ImageSearchResult](#SearchImages) [func SearchImages(query: string, limit: int, tlsVerify: ?bool, filter: ImageSearchFilter) ImageSearchResult](#SearchImages)
[func SendFile(type: string, length: int) string](#SendFile) [func SendFile(type: string, length: int) string](#SendFile)
@ -163,6 +163,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
[type ImageHistory](#ImageHistory) [type ImageHistory](#ImageHistory)
[type ImageSearchFilter](#ImageSearchFilter)
[type ImageSearchResult](#ImageSearchResult) [type ImageSearchResult](#ImageSearchResult)
[type InfoDistribution](#InfoDistribution) [type InfoDistribution](#InfoDistribution)
@ -1408,6 +1410,15 @@ tags [[]string](#[]string)
size [int](https://godoc.org/builtin#int) size [int](https://godoc.org/builtin#int)
comment [string](https://godoc.org/builtin#string) comment [string](https://godoc.org/builtin#string)
### <a name="ImageSearchFilter"></a>type ImageSearchFilter
Represents a filter for SearchImages
is_official [bool](https://godoc.org/builtin#bool)
is_automated [bool](https://godoc.org/builtin#bool)
star_count [int](https://godoc.org/builtin#int)
### <a name="ImageSearchResult"></a>type ImageSearchResult ### <a name="ImageSearchResult"></a>type ImageSearchResult
Represents a single search result from SearchImages Represents a single search result from SearchImages

View file

@ -1,19 +1,13 @@
package main package main
import ( import (
"context"
"reflect"
"strconv"
"strings" "strings"
"github.com/containers/image/docker"
"github.com/containers/image/types" "github.com/containers/image/types"
"github.com/containers/libpod/cmd/podman/cliconfig" "github.com/containers/libpod/cmd/podman/cliconfig"
"github.com/containers/libpod/cmd/podman/formats" "github.com/containers/libpod/cmd/podman/formats"
"github.com/containers/libpod/libpod/common" "github.com/containers/libpod/libpod/image"
sysreg "github.com/containers/libpod/pkg/registries"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -54,30 +48,6 @@ func init() {
flags.BoolVar(&searchCommand.TlsVerify, "tls-verify", true, "Require HTTPS and verify certificates when contacting registries (default: true)") flags.BoolVar(&searchCommand.TlsVerify, "tls-verify", true, "Require HTTPS and verify certificates when contacting registries (default: true)")
} }
type searchParams struct {
Index string
Name string
Description string
Stars int
Official string
Automated string
}
type searchOpts struct {
filter []string
limit int
noTrunc bool
format string
authfile string
insecureSkipTLSVerify types.OptionalBool
}
type searchFilterParams struct {
stars int
isAutomated *bool
isOfficial *bool
}
func searchCmd(c *cliconfig.SearchValues) error { func searchCmd(c *cliconfig.SearchValues) error {
args := c.InputArgs args := c.InputArgs
if len(args) > 1 { if len(args) > 1 {
@ -88,37 +58,29 @@ func searchCmd(c *cliconfig.SearchValues) error {
} }
term := args[0] term := args[0]
// Check if search term has a registry in it filter, err := image.ParseSearchFilter(c.Filter)
registry, err := sysreg.GetRegistry(term)
if err != nil { if err != nil {
return errors.Wrapf(err, "error getting registry from %q", term) return err
}
if registry != "" {
term = term[len(registry)+1:]
} }
format := genSearchFormat(c.Format) searchOptions := image.SearchOptions{
opts := searchOpts{ NoTrunc: c.NoTrunc,
format: format, Limit: c.Limit,
noTrunc: c.NoTrunc, Filter: *filter,
limit: c.Limit, Authfile: getAuthFile(c.Authfile),
filter: c.Filter,
authfile: getAuthFile(c.Authfile),
} }
if c.Flag("tls-verify").Changed { if c.Flag("tls-verify").Changed {
opts.insecureSkipTLSVerify = types.NewOptionalBool(!c.TlsVerify) searchOptions.InsecureSkipTLSVerify = types.NewOptionalBool(!c.TlsVerify)
} }
registries, err := getRegistries(registry)
results, err := image.SearchImages(term, searchOptions)
if err != nil { if err != nil {
return err return err
} }
format := genSearchFormat(c.Format)
filter, err := parseSearchFilter(&opts) out := formats.StdoutTemplateArray{Output: searchToGeneric(results), Template: format, Fields: results[0].HeaderMap()}
if err != nil { formats.Writer(out).Out()
return err return nil
}
return generateSearchOutput(term, registries, opts, *filter)
} }
func genSearchFormat(format string) string { func genSearchFormat(format string) string {
@ -130,175 +92,9 @@ func genSearchFormat(format string) string {
return "table {{.Index}}\t{{.Name}}\t{{.Description}}\t{{.Stars}}\t{{.Official}}\t{{.Automated}}\t" return "table {{.Index}}\t{{.Name}}\t{{.Description}}\t{{.Stars}}\t{{.Official}}\t{{.Automated}}\t"
} }
func searchToGeneric(params []searchParams) (genericParams []interface{}) { func searchToGeneric(params []image.SearchResult) (genericParams []interface{}) {
for _, v := range params { for _, v := range params {
genericParams = append(genericParams, interface{}(v)) genericParams = append(genericParams, interface{}(v))
} }
return genericParams return genericParams
} }
func (s *searchParams) headerMap() map[string]string {
v := reflect.Indirect(reflect.ValueOf(s))
values := make(map[string]string, v.NumField())
for i := 0; i < v.NumField(); i++ {
key := v.Type().Field(i).Name
value := key
values[key] = strings.ToUpper(splitCamelCase(value))
}
return values
}
// getRegistries returns the list of registries to search, depending on an optional registry specification
func getRegistries(registry string) ([]string, error) {
var registries []string
if registry != "" {
registries = append(registries, registry)
} else {
var err error
registries, err = sysreg.GetRegistries()
if err != nil {
return nil, errors.Wrapf(err, "error getting registries to search")
}
}
return registries, nil
}
func getSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) ([]searchParams, error) {
// Max number of queries by default is 25
limit := maxQueries
if opts.limit != 0 {
limit = opts.limit
}
sc := common.GetSystemContext("", opts.authfile, false)
sc.DockerInsecureSkipTLSVerify = opts.insecureSkipTLSVerify
sc.SystemRegistriesConfPath = sysreg.SystemRegistriesConfPath() // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place.
var paramsArr []searchParams
for _, reg := range registries {
results, err := docker.SearchRegistry(context.TODO(), sc, reg, term, limit)
if err != nil {
logrus.Errorf("error searching registry %q: %v", reg, err)
continue
}
index := reg
arr := strings.Split(reg, ".")
if len(arr) > 2 {
index = strings.Join(arr[len(arr)-2:], ".")
}
// limit is the number of results to output
// if the total number of results is less than the limit, output all
// if the limit has been set by the user, output those number of queries
limit := maxQueries
if len(results) < limit {
limit = len(results)
}
if opts.limit != 0 && opts.limit < len(results) {
limit = opts.limit
}
for i := 0; i < limit; i++ {
if len(opts.filter) > 0 {
// Check whether query matches filters
if !(matchesAutomatedFilter(filter, results[i]) && matchesOfficialFilter(filter, results[i]) && matchesStarFilter(filter, results[i])) {
continue
}
}
official := ""
if results[i].IsOfficial {
official = "[OK]"
}
automated := ""
if results[i].IsAutomated {
automated = "[OK]"
}
description := strings.Replace(results[i].Description, "\n", " ", -1)
if len(description) > 44 && !opts.noTrunc {
description = description[:descriptionTruncLength] + "..."
}
name := reg + "/" + results[i].Name
if index == "docker.io" && !strings.Contains(results[i].Name, "/") {
name = index + "/library/" + results[i].Name
}
params := searchParams{
Index: index,
Name: name,
Description: description,
Official: official,
Automated: automated,
Stars: results[i].StarCount,
}
paramsArr = append(paramsArr, params)
}
}
return paramsArr, nil
}
func generateSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) error {
searchOutput, err := getSearchOutput(term, registries, opts, filter)
if err != nil {
return err
}
if len(searchOutput) == 0 {
return nil
}
out := formats.StdoutTemplateArray{Output: searchToGeneric(searchOutput), Template: opts.format, Fields: searchOutput[0].headerMap()}
return formats.Writer(out).Out()
}
func parseSearchFilter(opts *searchOpts) (*searchFilterParams, error) {
filterParams := &searchFilterParams{}
ptrTrue := true
ptrFalse := false
for _, filter := range opts.filter {
arr := strings.Split(filter, "=")
switch arr[0] {
case "stars":
if len(arr) < 2 {
return nil, errors.Errorf("invalid `stars` filter %q, should be stars=<value>", filter)
}
stars, err := strconv.Atoi(arr[1])
if err != nil {
return nil, errors.Wrapf(err, "incorrect value type for stars filter")
}
filterParams.stars = stars
break
case "is-automated":
if len(arr) == 2 && arr[1] == "false" {
filterParams.isAutomated = &ptrFalse
} else {
filterParams.isAutomated = &ptrTrue
}
break
case "is-official":
if len(arr) == 2 && arr[1] == "false" {
filterParams.isOfficial = &ptrFalse
} else {
filterParams.isOfficial = &ptrTrue
}
break
default:
return nil, errors.Errorf("invalid filter type %q", filter)
}
}
return filterParams, nil
}
func matchesStarFilter(filter searchFilterParams, result docker.SearchResult) bool {
return result.StarCount >= filter.stars
}
func matchesAutomatedFilter(filter searchFilterParams, result docker.SearchResult) bool {
if filter.isAutomated != nil {
return result.IsAutomated == *filter.isAutomated
}
return true
}
func matchesOfficialFilter(filter searchFilterParams, result docker.SearchResult) bool {
if filter.isOfficial != nil {
return result.IsOfficial == *filter.isOfficial
}
return true
}

View file

@ -72,6 +72,12 @@ type ImageSearchResult (
star_count: int star_count: int
) )
type ImageSearchFilter (
is_official: ?bool,
is_automated: ?bool,
star_count: int
)
type Container ( type Container (
id: string, id: string,
image: string, image: string,
@ -681,7 +687,7 @@ method RemoveImage(name: string, force: bool) -> (image: string)
# SearchImages searches available registries for images that contain the # SearchImages searches available registries for images that contain the
# contents of "query" in their name. If "limit" is given, limits the amount of # contents of "query" in their name. If "limit" is given, limits the amount of
# search results per registry. # search results per registry.
method SearchImages(query: string, limit: ?int, tlsVerify: ?bool) -> (results: []ImageSearchResult) method SearchImages(query: string, limit: ?int, tlsVerify: ?bool, filter: ImageSearchFilter) -> (results: []ImageSearchResult)
# DeleteUnusedImages deletes any images not associated with a container. The IDs of the deleted images are returned # DeleteUnusedImages deletes any images not associated with a container. The IDs of the deleted images are returned
# in a string array. # in a string array.

277
libpod/image/search.go Normal file
View file

@ -0,0 +1,277 @@
package image
import (
"context"
"reflect"
"strconv"
"strings"
"sync"
"github.com/containers/image/docker"
"github.com/containers/image/types"
"github.com/containers/libpod/libpod/common"
sysreg "github.com/containers/libpod/pkg/registries"
"github.com/fatih/camelcase"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"golang.org/x/sync/semaphore"
)
const (
descriptionTruncLength = 44
maxQueries = 25
maxParallelSearches = int64(6)
)
// SearchResult is holding image-search related data.
type SearchResult struct {
// Index is the image index (e.g., "docker.io" or "quay.io")
Index string
// Name is the canoncical name of the image (e.g., "docker.io/library/alpine").
Name string
// Description of the image.
Description string
// Stars is the number of stars of the image.
Stars int
// Official indicates if it's an official image.
Official string
// Automated indicates if the image was created by an automated build.
Automated string
}
// SearchOptions are used to control the behaviour of SearchImages.
type SearchOptions struct {
// Filter allows to filter the results.
Filter SearchFilter
// Limit limits the number of queries per index (default: 25). Must be
// greater than 0 to overwrite the default value.
Limit int
// NoTrunc avoids the output to be truncated.
NoTrunc bool
// Authfile is the path to the authentication file.
Authfile string
// InsecureSkipTLSVerify allows to skip TLS verification.
InsecureSkipTLSVerify types.OptionalBool
}
// SearchFilter allows filtering the results of SearchImages.
type SearchFilter struct {
// Stars describes the minimal amount of starts of an image.
Stars int
// IsAutomated decides if only images from automated builds are displayed.
IsAutomated types.OptionalBool
// IsOfficial decides if only official images are displayed.
IsOfficial types.OptionalBool
}
func splitCamelCase(src string) string {
entries := camelcase.Split(src)
return strings.Join(entries, " ")
}
// HeaderMap returns the headers of a SearchResult.
func (s *SearchResult) HeaderMap() map[string]string {
v := reflect.Indirect(reflect.ValueOf(s))
values := make(map[string]string, v.NumField())
for i := 0; i < v.NumField(); i++ {
key := v.Type().Field(i).Name
value := key
values[key] = strings.ToUpper(splitCamelCase(value))
}
return values
}
// SearchImages searches images based on term and the specified SearchOptions
// in all registries.
func SearchImages(term string, options SearchOptions) ([]SearchResult, error) {
// Check if search term has a registry in it
registry, err := sysreg.GetRegistry(term)
if err != nil {
return nil, errors.Wrapf(err, "error getting registry from %q", term)
}
if registry != "" {
term = term[len(registry)+1:]
}
registries, err := getRegistries(registry)
if err != nil {
return nil, err
}
// searchOutputData is used as a return value for searching in parallel.
type searchOutputData struct {
data []SearchResult
err error
}
// Let's follow Firefox by limiting parallel downloads to 6.
sem := semaphore.NewWeighted(maxParallelSearches)
wg := sync.WaitGroup{}
wg.Add(len(registries))
data := make([]searchOutputData, len(registries))
searchImageInRegistryHelper := func(index int, registry string) {
defer sem.Release(1)
defer wg.Done()
searchOutput, err := searchImageInRegistry(term, registry, options)
data[index] = searchOutputData{data: searchOutput, err: err}
}
ctx := context.Background()
for i := range registries {
sem.Acquire(ctx, 1)
go searchImageInRegistryHelper(i, registries[i])
}
wg.Wait()
results := []SearchResult{}
for _, d := range data {
if d.err != nil {
return nil, d.err
}
results = append(results, d.data...)
}
return results, nil
}
// getRegistries returns the list of registries to search, depending on an optional registry specification
func getRegistries(registry string) ([]string, error) {
var registries []string
if registry != "" {
registries = append(registries, registry)
} else {
var err error
registries, err = sysreg.GetRegistries()
if err != nil {
return nil, errors.Wrapf(err, "error getting registries to search")
}
}
return registries, nil
}
func searchImageInRegistry(term string, registry string, options SearchOptions) ([]SearchResult, error) {
// Max number of queries by default is 25
limit := maxQueries
if options.Limit > 0 {
limit = options.Limit
}
sc := common.GetSystemContext("", options.Authfile, false)
sc.DockerInsecureSkipTLSVerify = options.InsecureSkipTLSVerify
// FIXME: Set this more globally. Probably no reason not to have it in
// every types.SystemContext, and to compute the value just once in one
// place.
sc.SystemRegistriesConfPath = sysreg.SystemRegistriesConfPath()
results, err := docker.SearchRegistry(context.TODO(), sc, registry, term, limit)
if err != nil {
logrus.Errorf("error searching registry %q: %v", registry, err)
return []SearchResult{}, nil
}
index := registry
arr := strings.Split(registry, ".")
if len(arr) > 2 {
index = strings.Join(arr[len(arr)-2:], ".")
}
// limit is the number of results to output
// if the total number of results is less than the limit, output all
// if the limit has been set by the user, output those number of queries
limit = maxQueries
if len(results) < limit {
limit = len(results)
}
if options.Limit != 0 && options.Limit < len(results) {
limit = options.Limit
}
paramsArr := []SearchResult{}
for i := 0; i < limit; i++ {
// Check whether query matches filters
if !(options.Filter.matchesAutomatedFilter(results[i]) && options.Filter.matchesOfficialFilter(results[i]) && options.Filter.matchesStarFilter(results[i])) {
continue
}
official := ""
if results[i].IsOfficial {
official = "[OK]"
}
automated := ""
if results[i].IsAutomated {
automated = "[OK]"
}
description := strings.Replace(results[i].Description, "\n", " ", -1)
if len(description) > 44 && !options.NoTrunc {
description = description[:descriptionTruncLength] + "..."
}
name := registry + "/" + results[i].Name
if index == "docker.io" && !strings.Contains(results[i].Name, "/") {
name = index + "/library/" + results[i].Name
}
params := SearchResult{
Index: index,
Name: name,
Description: description,
Official: official,
Automated: automated,
Stars: results[i].StarCount,
}
paramsArr = append(paramsArr, params)
}
return paramsArr, nil
}
// ParseSearchFilter turns the filter into a SearchFilter that can be used for
// searching images.
func ParseSearchFilter(filter []string) (*SearchFilter, error) {
sFilter := new(SearchFilter)
for _, f := range filter {
arr := strings.Split(f, "=")
switch arr[0] {
case "stars":
if len(arr) < 2 {
return nil, errors.Errorf("invalid `stars` filter %q, should be stars=<value>", filter)
}
stars, err := strconv.Atoi(arr[1])
if err != nil {
return nil, errors.Wrapf(err, "incorrect value type for stars filter")
}
sFilter.Stars = stars
break
case "is-automated":
if len(arr) == 2 && arr[1] == "false" {
sFilter.IsAutomated = types.OptionalBoolFalse
} else {
sFilter.IsAutomated = types.OptionalBoolTrue
}
break
case "is-official":
if len(arr) == 2 && arr[1] == "false" {
sFilter.IsOfficial = types.OptionalBoolFalse
} else {
sFilter.IsOfficial = types.OptionalBoolTrue
}
break
default:
return nil, errors.Errorf("invalid filter type %q", f)
}
}
return sFilter, nil
}
func (f *SearchFilter) matchesStarFilter(result docker.SearchResult) bool {
return result.StarCount >= f.Stars
}
func (f *SearchFilter) matchesAutomatedFilter(result docker.SearchResult) bool {
if f.IsAutomated != types.OptionalBoolUndefined {
return result.IsAutomated == (f.IsAutomated == types.OptionalBoolTrue)
}
return true
}
func (f *SearchFilter) matchesOfficialFilter(result docker.SearchResult) bool {
if f.IsOfficial != types.OptionalBoolUndefined {
return result.IsOfficial == (f.IsOfficial == types.OptionalBoolTrue)
}
return true
}

View file

@ -13,7 +13,6 @@ import (
"github.com/containers/buildah" "github.com/containers/buildah"
"github.com/containers/buildah/imagebuildah" "github.com/containers/buildah/imagebuildah"
"github.com/containers/image/docker"
dockerarchive "github.com/containers/image/docker/archive" dockerarchive "github.com/containers/image/docker/archive"
"github.com/containers/image/manifest" "github.com/containers/image/manifest"
"github.com/containers/image/transports/alltransports" "github.com/containers/image/transports/alltransports"
@ -22,7 +21,6 @@ import (
"github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/cmd/podman/varlink"
"github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod"
"github.com/containers/libpod/libpod/image" "github.com/containers/libpod/libpod/image"
sysreg "github.com/containers/libpod/pkg/registries"
"github.com/containers/libpod/pkg/util" "github.com/containers/libpod/pkg/util"
"github.com/containers/libpod/utils" "github.com/containers/libpod/utils"
"github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/archive"
@ -436,54 +434,53 @@ func (i *LibpodAPI) RemoveImage(call iopodman.VarlinkCall, name string, force bo
// SearchImages searches all registries configured in /etc/containers/registries.conf for an image // SearchImages searches all registries configured in /etc/containers/registries.conf for an image
// Requires an image name and a search limit as int // Requires an image name and a search limit as int
func (i *LibpodAPI) SearchImages(call iopodman.VarlinkCall, query string, limit *int64, tlsVerify *bool) error { func (i *LibpodAPI) SearchImages(call iopodman.VarlinkCall, query string, limit *int64, tlsVerify *bool, filter iopodman.ImageSearchFilter) error {
sc := image.GetSystemContext("", "", false) // Transform all arguments to proper types first
argLimit := 0
argTLSVerify := types.OptionalBoolUndefined
argIsOfficial := types.OptionalBoolUndefined
argIsAutomated := types.OptionalBoolUndefined
if limit != nil {
argLimit = int(*limit)
}
if tlsVerify != nil { if tlsVerify != nil {
sc.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!*tlsVerify) argTLSVerify = types.NewOptionalBool(!*tlsVerify)
}
if filter.Is_official != nil {
argIsOfficial = types.NewOptionalBool(*filter.Is_official)
}
if filter.Is_automated != nil {
argIsAutomated = types.NewOptionalBool(*filter.Is_automated)
} }
var registries []string
// Check if search query has a registry in it // Transform a SearchFilter the backend can deal with
registry, err := sysreg.GetRegistry(query) sFilter := image.SearchFilter{
IsOfficial: argIsOfficial,
IsAutomated: argIsAutomated,
Stars: int(filter.Star_count),
}
searchOptions := image.SearchOptions{
Limit: argLimit,
Filter: sFilter,
InsecureSkipTLSVerify: argTLSVerify,
}
results, err := image.SearchImages(query, searchOptions)
if err != nil { if err != nil {
return call.ReplyErrorOccurred(fmt.Sprintf("error getting registry from %q: %q", query, err)) return call.ReplyErrorOccurred(err.Error())
}
if registry != "" {
registries = append(registries, registry)
query = query[len(registry)+1:]
} else {
registries, err = sysreg.GetRegistries()
if err != nil {
return call.ReplyErrorOccurred(fmt.Sprintf("unable to get system registries: %q", err))
}
} }
var imageResults []iopodman.ImageSearchResult var imageResults []iopodman.ImageSearchResult
for _, reg := range registries { for _, result := range results {
var lim = 1000 i := iopodman.ImageSearchResult{
if limit != nil { Registry: result.Index,
lim = int(*limit) Description: result.Description,
} Is_official: result.Official == "[OK]",
results, err := docker.SearchRegistry(getContext(), sc, reg, query, lim) Is_automated: result.Automated == "[OK]",
if err != nil { Name: result.Name,
// If we are searching multiple registries, don't make something like an Star_count: int64(result.Stars),
// auth error fatal. Unfortunately we cannot differentiate between auth
// errors and other possibles errors
if len(registries) > 1 {
continue
}
return call.ReplyErrorOccurred(err.Error())
}
for _, result := range results {
i := iopodman.ImageSearchResult{
Registry: reg,
Description: result.Description,
Is_official: result.IsOfficial,
Is_automated: result.IsAutomated,
Name: result.Name,
Star_count: int64(result.StarCount),
}
imageResults = append(imageResults, i)
} }
imageResults = append(imageResults, i)
} }
return call.ReplySearchImages(imageResults) return call.ReplySearchImages(imageResults)
} }