container/podman
container/podman
1
0
Fork 0
mirror of https://github.com/containers/podman synced 2024-10-20 09:13:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Performance enhancement for podman images

Browse source 
Previous code was using slow routines to collect some of the information
needed to output images.  Specifically size was being calculated instead
of using the cached, already known size already available.  Also, straight-
lined several of the code paths.  Overall assessment is that these
improvements cut the time for images in half.

Signed-off-by: baude <bbaude@redhat.com>

Closes: #365
Approved by: mheon
This commit is contained in:
baude 2018-02-19 08:40:43 -06:00 committed by Atomic Bot
parent df7bf04dc6
commit 4929e37507
4 changed files with 391 additions and 194 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

335
cmd/podman/images.go
View file

@ -1,18 +1,15 @@
package main
import (
"fmt"
"reflect"
"strings"
"time"
"github.com/containers/storage"
"github.com/docker/go-units"
digest "github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/projectatomic/libpod/cmd/podman/formats"
"github.com/projectatomic/libpod/libpod"
"github.com/projectatomic/libpod/libpod/common"
"github.com/projectatomic/libpod/pkg/inspect"
"github.com/urfave/cli"
)
@ -31,15 +28,16 @@ type imagesJSONParams struct {
Name []string `json:"names"`
Digest digest.Digest `json:"digest"`
Created time.Time `json:"created"`
Size int64 `json:"size"`
Size *uint64 `json:"size"`
}
type imagesOptions struct {
quiet bool
noHeading bool
noTrunc bool
digests bool
format string
quiet bool
noHeading bool
noTrunc bool
digests bool
format string
outputformat string
}
var (
@ -64,7 +62,7 @@ var (
Name: "format",
Usage: "Change the output format to JSON or a Go template",
},
cli.StringFlag{
cli.StringSliceFlag{
Name: "filter, f",
Usage: "filter output based on conditions provided (default [])",
},
@ -92,38 +90,32 @@ func imagesCmd(c *cli.Context) error {
return errors.Wrapf(err, "Could not get runtime")
}
defer runtime.Shutdown(false)
var filterFuncs []libpod.ImageResultFilter
var imageInput string
if len(c.Args()) == 1 {
imageInput = c.Args().Get(0)
}
format := genImagesFormat(c.String("format"), c.Bool("quiet"), c.Bool("noheading"), c.Bool("digests"))
if len(c.Args()) > 1 {
return errors.New("'podman images' requires at most 1 argument")
}
if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 {
filterFuncs, err = CreateFilterFuncs(runtime, c, imageInput)
if err != nil {
return err
}
}
opts := imagesOptions{
quiet: c.Bool("quiet"),
noHeading: c.Bool("noheading"),
noTrunc: c.Bool("no-trunc"),
digests: c.Bool("digests"),
format: format,
format: c.String("format"),
}
var imageInput string
if len(c.Args()) == 1 {
imageInput = c.Args().Get(0)
}
if len(c.Args()) > 1 {
return errors.New("'podman images' requires at most 1 argument")
}
params, err := runtime.ParseImageFilter(imageInput, c.String("filter"))
if err != nil {
return errors.Wrapf(err, "error parsing filter")
}
// generate the different filters
labelFilter := generateImagesFilter(params, "label")
beforeImageFilter := generateImagesFilter(params, "before-image")
sinceImageFilter := generateImagesFilter(params, "since-image")
danglingFilter := generateImagesFilter(params, "dangling")
referenceFilter := generateImagesFilter(params, "reference")
imageInputFilter := generateImagesFilter(params, "image-input")
opts.outputformat = opts.setOutputFormat()
/*
podman does not implement --all for images
@ -131,29 +123,36 @@ func imagesCmd(c *cli.Context) error {
children to the image once built. until buildah supports caching builds,
it will not generate these intermediate images.
*/
images, err := runtime.GetImages(params, labelFilter, beforeImageFilter, sinceImageFilter, danglingFilter, referenceFilter, imageInputFilter)
images, err := runtime.GetImageResults()
if err != nil {
return errors.Wrapf(err, "could not get list of images matching filter")
return errors.Wrapf(err, "unable to get images")
}
return generateImagesOutput(runtime, images, opts)
var filteredImages []inspect.ImageResult
// filter the images
if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 {
filteredImages = libpod.FilterImages(images, filterFuncs)
} else {
filteredImages = images
}
return generateImagesOutput(runtime, filteredImages, opts)
}
func genImagesFormat(format string, quiet, noHeading, digests bool) string {
if format != "" {
func (i imagesOptions) setOutputFormat() string {
if i.format != "" {
// "\t" from the command line is not being recognized as a tab
// replacing the string "\t" to a tab character if the user passes in "\t"
return strings.Replace(format, `\t`, "\t", -1)
return strings.Replace(i.format, `\t`, "\t", -1)
}
if quiet {
if i.quiet {
return formats.IDString
}
format = "table {{.Repository}}\t{{.Tag}}\t"
if noHeading {
format := "table {{.Repository}}\t{{.Tag}}\t"
if i.noHeading {
format = "{{.Repository}}\t{{.Tag}}\t"
}
if digests {
if i.digests {
format += "{{.Digest}}\t"
}
format += "{{.ID}}\t{{.Created}}\t{{.Size}}\t"
@ -174,8 +173,65 @@ func imagesToGeneric(templParams []imagesTemplateParams, JSONParams []imagesJSON
return
}
// generate the header based on the template provided
func (i *imagesTemplateParams) headerMap() map[string]string {
// getImagesTemplateOutput returns the images information to be printed in human readable format
func getImagesTemplateOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) (imagesOutput []imagesTemplateParams) {
for _, img := range images {
createdTime := img.Created
imageID := "sha256:" + img.ID
if !opts.noTrunc {
imageID = shortID(img.ID)
}
params := imagesTemplateParams{
Repository: img.Repository,
Tag: img.Tag,
ID: imageID,
Digest: img.Digest,
Created: units.HumanDuration(time.Since((createdTime))) + " ago",
Size: units.HumanSizeWithPrecision(float64(*img.Size), 3),
}
imagesOutput = append(imagesOutput, params)
}
return
}
// getImagesJSONOutput returns the images information in its raw form
func getImagesJSONOutput(runtime *libpod.Runtime, images []inspect.ImageResult) (imagesOutput []imagesJSONParams) {
for _, img := range images {
params := imagesJSONParams{
ID: img.ID,
Name: img.RepoTags,
Digest: img.Digest,
Created: img.Created,
Size: img.Size,
}
imagesOutput = append(imagesOutput, params)
}
return
}
// generateImagesOutput generates the images based on the format provided
func generateImagesOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) error {
if len(images) == 0 {
return nil
}
var out formats.Writer
switch opts.format {
case formats.JSONString:
imagesOutput := getImagesJSONOutput(runtime, images)
out = formats.JSONStructArray{Output: imagesToGeneric([]imagesTemplateParams{}, imagesOutput)}
default:
imagesOutput := getImagesTemplateOutput(runtime, images, opts)
out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.outputformat, Fields: imagesOutput[0].HeaderMap()}
}
return formats.Writer(out).Out()
}
// HeaderMap produces a generic map of "headers" based on a line
// of output
func (i *imagesTemplateParams) HeaderMap() map[string]string {
v := reflect.Indirect(reflect.ValueOf(i))
values := make(map[string]string)
@ -190,157 +246,48 @@ func (i *imagesTemplateParams) headerMap() map[string]string {
return values
}
// getImagesTemplateOutput returns the images information to be printed in human readable format
func getImagesTemplateOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) (imagesOutput []imagesTemplateParams) {
var (
lastID string
)
for _, img := range images {
if opts.quiet && lastID == img.ID {
continue // quiet should not show the same ID multiple times
}
createdTime := img.Created
// CreateFilterFuncs returns an array of filter functions based on the user inputs
// and is later used to filter images for output
func CreateFilterFuncs(r *libpod.Runtime, c *cli.Context, userInput string) ([]libpod.ImageResultFilter, error) {
var filterFuncs []libpod.ImageResultFilter
for _, filter := range c.StringSlice("filter") {
splitFilter := strings.Split(filter, "=")
switch splitFilter[0] {
case "before":
before := r.NewImage(splitFilter[1])
_, beforeID, _ := before.GetLocalImageName()
imageID := "sha256:" + img.ID
if !opts.noTrunc {
imageID = shortID(img.ID)
}
repository := "<none>"
tag := "<none>"
if len(img.Names) > 0 {
arr := strings.Split(img.Names[0], ":")
repository = arr[0]
if len(arr) == 2 {
tag = arr[1]
if before.LocalName == "" {
return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1])
}
}
img, err := r.GetImage(beforeID)
if err != nil {
return nil, err
}
filterFuncs = append(filterFuncs, libpod.ImageCreatedBefore(img.Created))
case "after":
after := r.NewImage(splitFilter[1])
_, afterID, _ := after.GetLocalImageName()
imgData, _ := runtime.GetImageInspectInfo(*img)
if imgData != nil {
createdTime = *imgData.Created
if after.LocalName == "" {
return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1])
}
img, err := r.GetImage(afterID)
if err != nil {
return nil, err
}
filterFuncs = append(filterFuncs, libpod.ImageCreatedAfter(img.Created))
case "dangling":
filterFuncs = append(filterFuncs, libpod.ImageDangling())
case "label":
labelFilter := strings.Join(splitFilter[1:], "=")
filterFuncs = append(filterFuncs, libpod.ImageLabel(labelFilter))
default:
return nil, errors.Errorf("invalid filter %s ", splitFilter[0])
}
params := imagesTemplateParams{
Repository: repository,
Tag: tag,
ID: imageID,
Digest: imgData.Digest,
Created: units.HumanDuration(time.Since((createdTime))) + " ago",
Size: units.HumanSizeWithPrecision(float64(imgData.Size), 3),
}
imagesOutput = append(imagesOutput, params)
}
return
}
// getImagesJSONOutput returns the images information in its raw form
func getImagesJSONOutput(runtime *libpod.Runtime, images []*storage.Image) (imagesOutput []imagesJSONParams) {
for _, img := range images {
createdTime := img.Created
imgData, _ := runtime.GetImageInspectInfo(*img)
if imgData != nil {
createdTime = *imgData.Created
}
params := imagesJSONParams{
ID: img.ID,
Name: img.Names,
Digest: imgData.Digest,
Created: createdTime,
Size: imgData.Size,
}
imagesOutput = append(imagesOutput, params)
}
return
}
// generateImagesOutput generates the images based on the format provided
func generateImagesOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) error {
if len(images) == 0 {
return nil
}
var out formats.Writer
switch opts.format {
case formats.JSONString:
imagesOutput := getImagesJSONOutput(runtime, images)
out = formats.JSONStructArray{Output: imagesToGeneric([]imagesTemplateParams{}, imagesOutput)}
default:
imagesOutput := getImagesTemplateOutput(runtime, images, opts)
out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.format, Fields: imagesOutput[0].headerMap()}
}
return formats.Writer(out).Out()
}
// generateImagesFilter returns an ImageFilter based on filterType
// to add more filters, define a new case and write what the ImageFilter function should do
func generateImagesFilter(params *libpod.ImageFilterParams, filterType string) libpod.ImageFilter {
switch filterType {
case "label":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.Label == "" {
return true
}
pair := strings.SplitN(params.Label, "=", 2)
if val, ok := info.Labels[pair[0]]; ok {
if len(pair) == 2 && val == pair[1] {
return true
}
if len(pair) == 1 {
return true
}
}
return false
}
case "before-image":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.BeforeImage.IsZero() {
return true
}
return info.Created.Before(params.BeforeImage)
}
case "since-image":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.SinceImage.IsZero() {
return true
}
return info.Created.After(params.SinceImage)
}
case "dangling":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.Dangling == "" {
return true
}
if common.IsFalse(params.Dangling) && params.ImageName != "<none>" {
return true
}
if common.IsTrue(params.Dangling) && params.ImageName == "<none>" {
return true
}
return false
}
case "reference":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.ReferencePattern == "" {
return true
}
return libpod.MatchesReference(params.ImageName, params.ReferencePattern)
}
case "image-input":
return func(image *storage.Image, info *inspect.ImageData) bool {
if params == nil || params.ImageInput == "" {
return true
}
return libpod.MatchesReference(params.ImageName, params.ImageInput)
}
default:
fmt.Println("invalid filter type", filterType)
return nil
}
if len(strings.TrimSpace(userInput)) != 0 {
filterFuncs = append(filterFuncs, libpod.OutputImageFilter(userInput))
}
return filterFuncs, nil
}

178
libpod/runtime_img.go
View file

@ -108,6 +108,9 @@ type imageDecomposeStruct struct {
transport string
}
// ImageResultFilter is a mock function for image filtering
type ImageResultFilter func(inspect.ImageResult) bool
func (k *Image) assembleFqName() string {
return fmt.Sprintf("%s/%s:%s", k.Registry, k.ImageName, k.Tag)
}
@ -1262,3 +1265,178 @@ func getPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error
}
return policyContext, nil
}
// sizer knows its size.
type sizer interface {
Size() (int64, error)
}
func imageSize(img types.ImageSource) *uint64 {
if s, ok := img.(sizer); ok {
if sum, err := s.Size(); err == nil {
usum := uint64(sum)
return &usum
}
}
return nil
}
func reposToMap(repotags []string) map[string]string {
// map format is repo -> tag
repos := make(map[string]string)
for _, repo := range repotags {
var repository, tag string
if len(repo) > 0 {
li := strings.LastIndex(repo, ":")
repository = repo[0:li]
tag = repo[li+1:]
}
repos[repository] = tag
}
if len(repos) == 0 {
repos["<none>"] = "<none"
}
return repos
}
// GetImageResults gets the images for podman images and returns them as
// an array of ImageResults
func (r *Runtime) GetImageResults() ([]inspect.ImageResult, error) {
var results []inspect.ImageResult
images, err := r.store.Images()
if err != nil {
return nil, err
}
for _, image := range images {
storeRef, err := is.Transport.ParseStoreReference(r.store, image.ID)
if err != nil {
return nil, err
}
systemContext := &types.SystemContext{}
img, err := storeRef.NewImageSource(systemContext)
if err != nil {
return nil, err
}
ic, err := storeRef.NewImage(&types.SystemContext{})
if err != nil {
return nil, err
}
imgInspect, err := ic.Inspect()
if err != nil {
return nil, err
}
dangling := false
if len(image.Names) == 0 {
dangling = true
}
for repo, tag := range reposToMap(image.Names) {
results = append(results, inspect.ImageResult{
ID: image.ID,
Repository: repo,
RepoTags: image.Names,
Tag: tag,
Size: imageSize(img),
Digest: image.Digest,
Created: image.Created,
Labels: imgInspect.Labels,
Dangling: dangling,
})
}
}
return results, nil
}
// ImageCreatedBefore allows you to filter on images created before
// the given time.Time
func ImageCreatedBefore(createTime time.Time) ImageResultFilter {
return func(i inspect.ImageResult) bool {
if i.Created.Before(createTime) {
return true
}
return false
}
}
// ImageCreatedAfter allows you to filter on images created after
// the given time.Time
func ImageCreatedAfter(createTime time.Time) ImageResultFilter {
return func(i inspect.ImageResult) bool {
if i.Created.After(createTime) {
return true
}
return false
}
}
// ImageDangling allows you to filter images for dangling images
func ImageDangling() ImageResultFilter {
return func(i inspect.ImageResult) bool {
if i.Dangling {
return true
}
return false
}
}
// ImageLabel allows you to filter by images labels key and/or value
func ImageLabel(labelfilter string) ImageResultFilter {
// We need to handle both label=key and label=key=value
return func(i inspect.ImageResult) bool {
var value string
splitFilter := strings.Split(labelfilter, "=")
key := splitFilter[0]
if len(splitFilter) > 1 {
value = splitFilter[1]
}
for labelKey, labelValue := range i.Labels {
// handles label=key
if key == labelKey && len(strings.TrimSpace(value)) == 0 {
return true
}
//handles label=key=value
if key == labelKey && value == labelValue {
return true
}
}
return false
}
}
// OutputImageFilter allows you to filter by an a specific image name
func OutputImageFilter(name string) ImageResultFilter {
return func(i inspect.ImageResult) bool {
li := strings.LastIndex(name, ":")
var repository, tag string
if li < 0 {
repository = name
} else {
repository = name[0:li]
tag = name[li+1:]
}
if repository == i.Repository && len(strings.TrimSpace(tag)) == 0 {
return true
}
if repository == i.Repository && tag == i.Tag {
return true
}
return false
}
}
// FilterImages filters images using a set of predefined fitler funcs
func FilterImages(images []inspect.ImageResult, filters []ImageResultFilter) []inspect.ImageResult {
var filteredImages []inspect.ImageResult
for _, image := range images {
include := true
for _, filter := range filters {
include = include && filter(image)
}
if include {
filteredImages = append(filteredImages, image)
}
}
return filteredImages
}

15
pkg/inspect/inspect.go
View file

@ -202,3 +202,18 @@ type NetworkSettings struct {
IPv6Gateway string `json:"IPv6Gateway"`
MacAddress string `json:"MacAddress"`
}
// ImageResult is used for podman images for collection and output
type ImageResult struct {
Tag string
Repository string
RepoDigests []string
RepoTags []string
ID string
Digest digest.Digest
ConfigDigest digest.Digest
Created time.Time
Size *uint64
Labels map[string]string
Dangling bool
}

57
test/e2e/images_test.go
View file

@ -37,6 +37,15 @@ var _ = Describe("Podman images", func() {
Expect(session.LineInOuputStartsWith("docker.io/library/busybox")).To(BeTrue())
})
It("podman images with digests", func() {
session := podmanTest.Podman([]string{"images", "--digests"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
Expect(len(session.OutputToStringArray())).To(BeNumerically(">", 2))
Expect(session.LineInOuputStartsWith("docker.io/library/alpine")).To(BeTrue())
Expect(session.LineInOuputStartsWith("docker.io/library/busybox")).To(BeTrue())
})
It("podman images in JSON format", func() {
session := podmanTest.Podman([]string{"images", "--format=json"})
session.WaitWithDefaultTimeout()
@ -44,10 +53,58 @@ var _ = Describe("Podman images", func() {
Expect(session.IsJSONOutputValid()).To(BeTrue())
})
It("podman images in GO template format", func() {
session := podmanTest.Podman([]string{"images", "--format={{.ID}}"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
})
It("podman images with short options", func() {
session := podmanTest.Podman([]string{"images", "-qn"})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
Expect(len(session.OutputToStringArray())).To(BeNumerically(">", 1))
})
It("podman images filter by image name", func() {
session := podmanTest.Podman([]string{"images", "-q", ALPINE})
session.WaitWithDefaultTimeout()
Expect(session.ExitCode()).To(Equal(0))
Expect(len(session.OutputToStringArray())).To(Equal(1))
})
It("podman images filter before image", func() {
dockerfile := `FROM docker.io/library/alpine:latest
`
podmanTest.BuildImage(dockerfile, "foobar.com/before:latest")
result := podmanTest.Podman([]string{"images", "-q", "-f", "before=foobar.com/before:latest"})
result.WaitWithDefaultTimeout()
Expect(result.ExitCode()).To(Equal(0))
Expect(len(result.OutputToStringArray())).To(Equal(2))
})
It("podman images filter after image", func() {
rmi := podmanTest.Podman([]string{"rmi", "busybox"})
rmi.WaitWithDefaultTimeout()
Expect(rmi.ExitCode()).To(Equal(0))
dockerfile := `FROM docker.io/library/alpine:latest
`
podmanTest.BuildImage(dockerfile, "foobar.com/before:latest")
result := podmanTest.Podman([]string{"images", "-q", "-f", "after=docker.io/library/alpine:latest"})
result.WaitWithDefaultTimeout()
Expect(result.ExitCode()).To(Equal(0))
Expect(len(result.OutputToStringArray())).To(Equal(1))
})
It("podman images filter dangling", func() {
dockerfile := `FROM docker.io/library/alpine:latest
`
podmanTest.BuildImage(dockerfile, "foobar.com/before:latest")
podmanTest.BuildImage(dockerfile, "foobar.com/before:latest")
result := podmanTest.Podman([]string{"images", "-q", "-f", "dangling=true"})
result.WaitWithDefaultTimeout()
Expect(result.ExitCode()).To(Equal(0))
Expect(len(result.OutputToStringArray())).To(Equal(1))
})
})