Bump github.com/containers/storage from 1.21.2 to 1.23.0

Bumps [github.com/containers/storage](https://github.com/containers/storage) from 1.21.2 to 1.23.0.
- [Release notes](https://github.com/containers/storage/releases)
- [Changelog](https://github.com/containers/storage/blob/master/docs/containers-storage-changes.md)
- [Commits](https://github.com/containers/storage/compare/v1.21.2...v1.23.0)

Signed-off-by: dependabot-preview[bot] <support@dependabot.com>
Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/containers/conmon v2.0.19+incompatible
 	github.com/containers/image/v5 v5.5.1
 	github.com/containers/psgo v1.5.1
-	github.com/containers/storage v1.21.2
+	github.com/containers/storage v1.23.0
 	github.com/coreos/go-systemd/v22 v22.1.0
 	github.com/cri-o/ocicni v0.2.0
 	github.com/cyphar/filepath-securejoin v0.2.2

go.sum

@@ -90,6 +90,8 @@ github.com/containers/psgo v1.5.1/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzP
 github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc=
 github.com/containers/storage v1.21.2 h1:bf9IqA+g6ClBviqVG5lVCp5tTH9lvWwjYws7mVYSti0=
 github.com/containers/storage v1.21.2/go.mod h1:I1EIAA7B4OwWRSA0b4yq2AW1wjvvfcY0zLWQuwTa4zw=
+github.com/containers/storage v1.23.0 h1:gYyNkBiihC2FvGiHOjOjpnfojYwgxpLVooTUlmD6pxs=
+github.com/containers/storage v1.23.0/go.mod h1:I1EIAA7B4OwWRSA0b4yq2AW1wjvvfcY0zLWQuwTa4zw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38=

vendor/github.com/containers/storage/.cirrus.yml

@@ -62,8 +62,10 @@ testing_task:
         - lint
 
     # Not all $TEST_DRIVER combinations are valid for all OS types.
-    # Note: Nested-variable resolution happens at runtime, not eval. time.
-    # Use verbose logic for ease of reading/maintaining.
+    # N/B: As of the addition of this note, nested-variable resolution
+    # does not happen for boolean `only_if` expressions.  Since $VM_IMAGE
+    # contains nested variables, we must filter based on that and not the
+    # actual distro/version value.
     only_if: >-
         ( $VM_IMAGE =~ '.*UBUNTU.*' && $TEST_DRIVER == "vfs" ) ||
         ( $VM_IMAGE =~ '.*UBUNTU.*' && $TEST_DRIVER == "aufs" ) ||
@@ -146,3 +148,15 @@ vendor_task:
         folder: $GOPATH/pkg/mod
     build_script: make vendor
     test_script: hack/tree_status.sh
+
+# Represent overall pass/fail status from required dependent tasks
+success_task:
+    depends_on:
+        - lint
+        - testing
+        - meta
+        - vendor
+    container:
+        image: golang:1.14
+    clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"'  # Source code not needed
+    script: /bin/true

vendor/github.com/containers/storage/VERSION

@@ -1 +1 @@
-1.21.2
+1.23.1-dev

vendor/github.com/containers/storage/images.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/containers/storage/pkg/ioutils"
 	"github.com/containers/storage/pkg/stringid"
+	"github.com/containers/storage/pkg/stringutils"
 	"github.com/containers/storage/pkg/truncindex"
 	digest "github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
@@ -465,6 +466,19 @@ func (r *imageStore) addMappedTopLayer(id, layer string) error {
 	return errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", id)
 }
 
+func (r *imageStore) removeMappedTopLayer(id, layer string) error {
+	if image, ok := r.lookup(id); ok {
+		initialLen := len(image.MappedTopLayers)
+		image.MappedTopLayers = stringutils.RemoveFromSlice(image.MappedTopLayers, layer)
+		// No layer was removed.  No need to save.
+		if initialLen == len(image.MappedTopLayers) {
+			return nil
+		}
+		return r.Save()
+	}
+	return errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", id)
+}
+
 func (r *imageStore) Metadata(id string) (string, error) {
 	if image, ok := r.lookup(id); ok {
 		return image.Metadata, nil

vendor/github.com/containers/storage/pkg/archive/archive.go

@@ -602,7 +602,7 @@ func (ta *tarAppender) addTarFile(path, name string) error {
 	return nil
 }
 
-func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns, ignoreChownErrors bool) error {
+func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns, ignoreChownErrors bool, buffer []byte) error {
 	// hdr.Mode is in linux format, which we can use for sycalls,
 	// but for os.Foo() calls we need the mode converted to os.FileMode,
 	// so use hdrInfo.Mode() (they differ for e.g. setuid bits)
@@ -626,7 +626,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L
 		if err != nil {
 			return err
 		}
-		if _, err := io.Copy(file, reader); err != nil {
+		if _, err := io.CopyBuffer(file, reader, buffer); err != nil {
 			file.Close()
 			return err
 		}
@@ -942,6 +942,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err
 	idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps)
 	rootIDs := idMappings.RootPair()
 	whiteoutConverter := getWhiteoutConverter(options.WhiteoutFormat, options.WhiteoutData)
+	buffer := make([]byte, 1<<20)
 
 	// Iterate through the files in the archive.
 loop:
@@ -1038,7 +1039,7 @@ loop:
 			chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
 		}
 
-		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, chownOpts, options.InUserNS, options.IgnoreChownErrors); err != nil {
+		if err := createTarFile(path, dest, hdr, trBuf, !options.NoLchown, chownOpts, options.InUserNS, options.IgnoreChownErrors, buffer); err != nil {
 			return err
 		}
 

vendor/github.com/containers/storage/pkg/archive/changes_unix.go

@@ -18,9 +18,11 @@ func statDifferent(oldStat *system.StatT, oldInfo *FileInfo, newStat *system.Sta
 	if cuid, cgid, err := newInfo.idMappings.ToContainer(idtools.IDPair{UID: int(uid), GID: int(gid)}); err == nil {
 		uid = uint32(cuid)
 		gid = uint32(cgid)
-		if oldcuid, oldcgid, err := oldInfo.idMappings.ToContainer(idtools.IDPair{UID: int(oldUID), GID: int(oldGID)}); err == nil {
-			oldUID = uint32(oldcuid)
-			oldGID = uint32(oldcgid)
+		if oldInfo != nil {
+			if oldcuid, oldcgid, err := oldInfo.idMappings.ToContainer(idtools.IDPair{UID: int(oldUID), GID: int(oldGID)}); err == nil {
+				oldUID = uint32(oldcuid)
+				oldGID = uint32(oldcgid)
+			}
 		}
 	}
 	ownerChanged := uid != oldUID || gid != oldGID

vendor/github.com/containers/storage/pkg/archive/diff.go

@@ -37,6 +37,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 
 	aufsTempdir := ""
 	aufsHardlinks := make(map[string]*tar.Header)
+	buffer := make([]byte, 1<<20)
 
 	// Iterate through the files in the archive.
 	for {
@@ -105,7 +106,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 					}
 					defer os.RemoveAll(aufsTempdir)
 				}
-				if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true, nil, options.InUserNS, options.IgnoreChownErrors); err != nil {
+				if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, true, nil, options.InUserNS, options.IgnoreChownErrors, buffer); err != nil {
 					return 0, err
 				}
 			}
@@ -196,7 +197,7 @@ func UnpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64,
 				return 0, err
 			}
 
-			if err := createTarFile(path, dest, srcHdr, srcData, true, nil, options.InUserNS, options.IgnoreChownErrors); err != nil {
+			if err := createTarFile(path, dest, srcHdr, srcData, true, nil, options.InUserNS, options.IgnoreChownErrors, buffer); err != nil {
 				return 0, err
 			}
 

vendor/github.com/containers/storage/pkg/fileutils/fileutils.go

@@ -262,7 +262,7 @@ func (p *Pattern) compile() error {
 		}
 	}
 
-	regStr += "(/.*)?$"
+	regStr += "(" + escSL + ".*)?$"
 
 	re, err := regexp.Compile(regStr)
 	if err != nil {

vendor/github.com/containers/storage/pkg/stringutils/stringutils.go

@@ -56,13 +56,24 @@ func Truncate(s string, maxlen int) string {
 // Comparison is case insensitive
 func InSlice(slice []string, s string) bool {
 	for _, ss := range slice {
-		if strings.ToLower(s) == strings.ToLower(ss) {
+		if strings.EqualFold(s, ss) {
 			return true
 		}
 	}
 	return false
 }
 
+// RemoveFromSlice removes a string from a slice.  The string can be present
+// multiple times.  The entire slice is iterated.
+func RemoveFromSlice(slice []string, s string) (ret []string) {
+	for _, ss := range slice {
+		if !strings.EqualFold(s, ss) {
+			ret = append(ret, ss)
+		}
+	}
+	return ret
+}
+
 func quote(word string, buf *bytes.Buffer) {
 	// Bail out early for "simple" strings
 	if word != "" && !strings.ContainsAny(word, "\\'\"`${[|&;<>()~*?! \t\n") {

vendor/github.com/containers/storage/pkg/unshare/unshare.c

@@ -292,6 +292,7 @@ static int containers_reexec(int flags) {
 		fprintf(stderr, "Error during reexec(...): %m\n");
 		return -1;
 	}
+	close(fd);
 	return 0;
 }
 

vendor/github.com/containers/storage/storage_test.conf

@@ -0,0 +1,35 @@
+# This file is is a TEST configuration file for all tools
+# that use the containers/storage library.
+# See man 5 containers-storage.conf for more information
+# The "container storage" table contains all of the server options.
+[storage]
+
+# Default Storage Driver
+driver = ""
+
+# Temporary storage location
+runroot = "$HOME/$UID/containers/storage"
+
+# Primary Read/Write location of container storage
+graphroot = "$HOME/$UID/containers/storage"
+
+# Storage path for rootless users
+#
+rootless_storage_path = "$HOME/$UID/containers/storage"
+
+[storage.options]
+# Storage options to be passed to underlying storage drivers
+
+# AdditionalImageStores is used to pass paths to additional Read/Only image stores
+# Must be comma separated list.
+additionalimagestores = [
+]
+
+[storage.options.overlay]
+
+# mountopt specifies comma separated list of extra mount options
+mountopt = "nodev"
+
+
+[storage.options.thinpool]
+# Storage Options for thinpool

vendor/github.com/containers/storage/store.go

@@ -2223,16 +2223,23 @@ func (s *store) DeleteLayer(id string) error {
 		}
 		for _, layer := range layers {
 			if layer.Parent == id {
-				return ErrLayerHasChildren
+				return errors.Wrapf(ErrLayerHasChildren, "used by layer %v", layer.ID)
 			}
 		}
 		images, err := ristore.Images()
 		if err != nil {
 			return err
 		}
+
 		for _, image := range images {
-			if image.TopLayer == id || stringutils.InSlice(image.MappedTopLayers, id) {
-				return errors.Wrapf(ErrLayerUsedByImage, "Layer %v used by image %v", id, image.ID)
+			if image.TopLayer == id {
+				return errors.Wrapf(ErrLayerUsedByImage, "layer %v used by image %v", id, image.ID)
+			}
+			if stringutils.InSlice(image.MappedTopLayers, id) {
+				// No write access to the image store, fail before the layer is deleted
+				if _, ok := ristore.(*imageStore); !ok {
+					return errors.Wrapf(ErrLayerUsedByImage, "layer %v used by image %v", id, image.ID)
+				}
 			}
 		}
 		containers, err := rcstore.Containers()
@@ -2241,10 +2248,25 @@ func (s *store) DeleteLayer(id string) error {
 		}
 		for _, container := range containers {
 			if container.LayerID == id {
-				return errors.Wrapf(ErrLayerUsedByContainer, "Layer %v used by container %v", id, container.ID)
+				return errors.Wrapf(ErrLayerUsedByContainer, "layer %v used by container %v", id, container.ID)
 			}
 		}
-		return rlstore.Delete(id)
+		if err := rlstore.Delete(id); err != nil {
+			return errors.Wrapf(err, "delete layer %v", id)
+		}
+
+		// The check here is used to avoid iterating the images if we don't need to.
+		// There is already a check above for the imageStore to be writeable when the layer is part of MappedTopLayers.
+		if istore, ok := ristore.(*imageStore); ok {
+			for _, image := range images {
+				if stringutils.InSlice(image.MappedTopLayers, id) {
+					if err = istore.removeMappedTopLayer(image.ID, id); err != nil {
+						return errors.Wrapf(err, "remove mapped top layer %v from image %v", id, image.ID)
+					}
+				}
+			}
+		}
+		return nil
 	}
 	return ErrNotALayer
 }

vendor/github.com/containers/storage/userns.go

@@ -252,7 +252,7 @@ func subtractHostIDs(avail idtools.IDMap, used idtools.IDMap) []idtools.IDMap {
 		}
 		r2 := idtools.IDMap{
 			ContainerID: used.ContainerID + used.Size,
-			HostID:      used.HostID + used.Size,
+			HostID:      avail.HostID + (used.HostID - avail.HostID),
 			Size:        avail.HostID + avail.Size - used.HostID - used.Size,
 		}
 		return []idtools.IDMap{r1, r2}
@@ -297,7 +297,7 @@ func subtractContainerIDs(avail idtools.IDMap, used idtools.IDMap) []idtools.IDM
 		}
 		r2 := idtools.IDMap{
 			ContainerID: used.ContainerID + used.Size,
-			HostID:      avail.HostID + used.Size,
+			HostID:      avail.HostID + (used.ContainerID - avail.ContainerID),
 			Size:        avail.ContainerID + avail.Size - used.ContainerID - used.Size,
 		}
 		return []idtools.IDMap{r1, r2}
@@ -314,22 +314,17 @@ func subtractContainerIDs(avail idtools.IDMap, used idtools.IDMap) []idtools.IDM
 // subtractAll subtracts all usedIDs from the available IDs.
 func subtractAll(availableIDs, usedIDs []idtools.IDMap, host bool) []idtools.IDMap {
 	for _, u := range usedIDs {
-		for i := 0; i < len(availableIDs); {
-			var prev []idtools.IDMap
-			if i > 0 {
-				prev = availableIDs[:i-1]
-			}
-			next := availableIDs[i+1:]
-			cur := availableIDs[i]
+		var newAvailableIDs []idtools.IDMap
+		for _, cur := range availableIDs {
 			var newRanges []idtools.IDMap
 			if host {
 				newRanges = subtractHostIDs(cur, u)
 			} else {
 				newRanges = subtractContainerIDs(cur, u)
 			}
-			availableIDs = append(append(prev, newRanges...), next...)
-			i += len(newRanges)
+			newAvailableIDs = append(newAvailableIDs, newRanges...)
 		}
+		availableIDs = newAvailableIDs
 	}
 	return availableIDs
 }
@@ -361,6 +356,7 @@ func findAvailableIDRange(size uint32, availableIDs, usedIDs []idtools.IDMap) ([
 			return avail[:i+1], nil
 		}
 		remaining -= uint32(avail[i].Size)
+		currentID += avail[i].Size
 	}
 
 	return nil, errors.New("could not find enough available IDs")
@@ -452,6 +448,5 @@ func (s *store) getAutoUserNS(id string, options *AutoUserNsOptions, image *Imag
 	if len(options.AdditionalGIDMappings) > 0 {
 		availableGIDs = subtractAll(availableGIDs, options.AdditionalGIDMappings, false)
 	}
-
 	return append(availableUIDs, options.AdditionalUIDMappings...), append(availableGIDs, options.AdditionalGIDMappings...), nil
 }

vendor/github.com/containers/storage/utils.go

@@ -5,9 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"os/user"
 	"path/filepath"
-	"regexp"
 	"strconv"
 	"strings"
 
@@ -234,8 +232,9 @@ func DefaultStoreOptionsAutoDetectUID() (StoreOptions, error) {
 	return DefaultStoreOptions(uid != 0, uid)
 }
 
-// DefaultStoreOptions returns the default storage ops for containers
-func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
+// defaultStoreOptionsIsolated is an internal implementation detail of DefaultStoreOptions to allow testing.
+// Everyone but the tests this is intended for should only call DefaultStoreOptions, never this function.
+func defaultStoreOptionsIsolated(rootless bool, rootlessUID int, storageConf string) (StoreOptions, error) {
 	var (
 		defaultRootlessRunRoot   string
 		defaultRootlessGraphRoot string
@@ -248,11 +247,6 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
 			return storageOpts, err
 		}
 	}
-
-	storageConf, err := DefaultConfigFile(rootless && rootlessUID != 0)
-	if err != nil {
-		return storageOpts, err
-	}
 	_, err = os.Stat(storageConf)
 	if err != nil && !os.IsNotExist(err) {
 		return storageOpts, errors.Wrapf(err, "cannot stat %s", storageConf)
@@ -263,6 +257,20 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
 		storageOpts = StoreOptions{}
 		reloadConfigurationFileIfNeeded(storageConf, &storageOpts)
 	}
+	if storageOpts.RunRoot != "" {
+		runRoot, err := expandEnvPath(storageOpts.RunRoot, rootlessUID)
+		if err != nil {
+			return storageOpts, err
+		}
+		storageOpts.RunRoot = runRoot
+	}
+	if storageOpts.GraphRoot != "" {
+		graphRoot, err := expandEnvPath(storageOpts.GraphRoot, rootlessUID)
+		if err != nil {
+			return storageOpts, err
+		}
+		storageOpts.GraphRoot = graphRoot
+	}
 
 	if rootless && rootlessUID != 0 {
 		if err == nil {
@@ -276,16 +284,10 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
 				storageOpts.GraphRoot = defaultRootlessGraphRoot
 			}
 			if storageOpts.RootlessStoragePath != "" {
-				if err = validRootlessStoragePathFormat(storageOpts.RootlessStoragePath); err != nil {
-					return storageOpts, err
-				}
-				rootlessStoragePath := strings.Replace(storageOpts.RootlessStoragePath, "$HOME", homedir.Get(), -1)
-				rootlessStoragePath = strings.Replace(rootlessStoragePath, "$UID", strconv.Itoa(rootlessUID), -1)
-				usr, err := user.LookupId(strconv.Itoa(rootlessUID))
+				rootlessStoragePath, err := expandEnvPath(storageOpts.RootlessStoragePath, rootlessUID)
 				if err != nil {
 					return storageOpts, err
 				}
-				rootlessStoragePath = strings.Replace(rootlessStoragePath, "$USER", usr.Username, -1)
 				storageOpts.GraphRoot = rootlessStoragePath
 			}
 		}
@@ -293,22 +295,19 @@ func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
 	return storageOpts, nil
 }
 
-// validRootlessStoragePathFormat checks if the environments contained in the path are accepted
-func validRootlessStoragePathFormat(path string) error {
-	if !strings.Contains(path, "$") {
-		return nil
+// DefaultStoreOptions returns the default storage ops for containers
+func DefaultStoreOptions(rootless bool, rootlessUID int) (StoreOptions, error) {
+	storageConf, err := DefaultConfigFile(rootless && rootlessUID != 0)
+	if err != nil {
+		return defaultStoreOptions, err
 	}
+	return defaultStoreOptionsIsolated(rootless, rootlessUID, storageConf)
+}
 
-	splitPaths := strings.SplitAfter(path, "$")
-	validEnv := regexp.MustCompile(`^(HOME|USER|UID)([^a-zA-Z]|$)`).MatchString
-	if len(splitPaths) > 1 {
-		for _, p := range splitPaths[1:] {
-			if !validEnv(p) {
-				return errors.Errorf("Unrecognized environment variable")
-			}
-		}
-	}
-	return nil
+func expandEnvPath(path string, rootlessUID int) (string, error) {
+	path = strings.Replace(path, "$UID", strconv.Itoa(rootlessUID), -1)
+	path = os.ExpandEnv(path)
+	return path, nil
 }
 
 func validateMountOptions(mountOptions []string) error {

vendor/modules.txt

@@ -156,7 +156,7 @@ github.com/containers/psgo/internal/dev
 github.com/containers/psgo/internal/host
 github.com/containers/psgo/internal/proc
 github.com/containers/psgo/internal/process
-# github.com/containers/storage v1.21.2
+# github.com/containers/storage v1.23.0
 github.com/containers/storage
 github.com/containers/storage/drivers
 github.com/containers/storage/drivers/aufs