fix: minor allocation improvements in xlMetaV2 (#12133)

cmd/gateway/gcs/gateway-gcs_test.go

@@ -190,13 +190,12 @@ func TestFromMinioClientListBucketResultToV2Info(t *testing.T) {
 
 // Test for gcsParseProjectID
 func TestGCSParseProjectID(t *testing.T) {
-	f, err := ioutil.TempFile("", "")
+	f, err := ioutil.TempFile("", "TestGCSParseProjectID-*")
 	if err != nil {
 		t.Error(err)
 		return
 	}
 	defer os.Remove(f.Name())
-	defer f.Close()
 
 	contents := `
 {
@@ -205,6 +204,7 @@ func TestGCSParseProjectID(t *testing.T) {
 }
 `
 	f.WriteString(contents)
+	f.Close()
 	projectID, err := gcsParseProjectID(f.Name())
 	if err != nil {
 		t.Fatal(err)
@@ -217,8 +217,20 @@ func TestGCSParseProjectID(t *testing.T) {
 		t.Errorf(`Expected to fail but succeeded reading "non-existent"`)
 	}
 
-	f.WriteString(`,}`)
+	contents = `
-
+{
+  "type": "service_account",
+  "project_id": "miniotesting"
+},}
+`
+	f, err = ioutil.TempFile("", "TestGCSParseProjectID-*")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer os.Remove(f.Name())
+	f.WriteString(contents)
+	f.Close()
 	if _, err := gcsParseProjectID(f.Name()); err == nil {
 		t.Errorf(`Expected to fail reading corrupted credentials file`)
 	}

cmd/storage-rest-server.go

@@ -852,6 +852,8 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse {
 // The returned reader contains the payload and must be closed if no error is returned.
 func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
 	var tmp [1]byte
+	// 8K copy buffer, reused for less allocs...
+	var buf [8 << 10]byte
 	for {
 		_, err := io.ReadFull(respBody, tmp[:])
 		if err != nil {
@@ -861,7 +863,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
 		switch tmp[0] {
 		case 0:
 			// 0 is unbuffered, copy the rest.
-			_, err := io.Copy(w, respBody)
+			_, err := io.CopyBuffer(w, respBody, buf[:])
 			if err == io.EOF {
 				return nil
 			}
@@ -880,7 +882,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error {
 				return err
 			}
 			length := binary.LittleEndian.Uint32(tmp[:])
-			_, err = io.CopyN(w, respBody, int64(length))
+			_, err = io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf[:])
 			if err != nil {
 				return err
 			}

cmd/xl-storage-format-utils.go

@@ -18,21 +18,24 @@
 package cmd
 
 import (
+	"sort"
+
 	jsoniter "github.com/json-iterator/go"
 )
 
+// versionsSorter sorts FileInfo slices by version.
 type versionsSorter []FileInfo
 
-func (v versionsSorter) Len() int      { return len(v) }
+func (v versionsSorter) sort() {
-func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] }
+	sort.Slice(v, func(i, j int) bool {
-func (v versionsSorter) Less(i, j int) bool {
+		if v[i].IsLatest {
-	if v[i].IsLatest {
+			return true
-		return true
+		}
-	}
+		if v[j].IsLatest {
-	if v[j].IsLatest {
+			return false
-		return false
+		}
-	}
+		return v[i].ModTime.After(v[j].ModTime)
-	return v[i].ModTime.After(v[j].ModTime)
+	})
 }
 
 func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) {

cmd/xl-storage-format-v2.go

@@ -524,13 +524,35 @@ func (x *xlMetaInlineData) rename(oldKey, newKey string) bool {
 	return true
 }
 
-// remove will remove a key.
+// remove will remove one or more keys.
-// Returns whether the key was found.
+// Returns true if any key was found.
-func (x *xlMetaInlineData) remove(key string) bool {
+func (x *xlMetaInlineData) remove(keys ...string) bool {
 	in := x.afterVersion()
 	sz, buf, _ := msgp.ReadMapHeaderBytes(in)
-	keys := make([][]byte, 0, sz)
+	newKeys := make([][]byte, 0, sz)
-	vals := make([][]byte, 0, sz)
+	newVals := make([][]byte, 0, sz)
+	var removeKey func(s []byte) bool
+
+	// Copy if big number of compares...
+	if len(keys) > 5 && sz > 5 {
+		mKeys := make(map[string]struct{}, len(keys))
+		for _, key := range keys {
+			mKeys[key] = struct{}{}
+		}
+		removeKey = func(s []byte) bool {
+			_, ok := mKeys[string(s)]
+			return ok
+		}
+	} else {
+		removeKey = func(s []byte) bool {
+			for _, key := range keys {
+				if key == string(s) {
+					return true
+				}
+			}
+			return false
+		}
+	}
 
 	// Version plus header...
 	plSize := 1 + msgp.MapHeaderSize
@@ -546,10 +568,10 @@ func (x *xlMetaInlineData) remove(key string) bool {
 		if err != nil {
 			break
 		}
-		if string(foundKey) != key {
+		if !removeKey(foundKey) {
 			plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal)
-			keys = append(keys, foundKey)
+			newKeys = append(newKeys, foundKey)
-			vals = append(vals, foundVal)
+			newVals = append(newVals, foundVal)
 		} else {
 			found = true
 		}
@@ -559,13 +581,13 @@ func (x *xlMetaInlineData) remove(key string) bool {
 		return false
 	}
 	// If none left...
-	if len(keys) == 0 {
+	if len(newKeys) == 0 {
 		*x = nil
 		return true
 	}
 
 	// Reserialize...
-	x.serialize(plSize, keys, vals)
+	x.serialize(plSize, newKeys, newVals)
 	return true
 }
 
@@ -891,11 +913,6 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 	return nil
 }
 
-func newXLMetaV2(fi FileInfo) (xlMetaV2, error) {
-	xlMeta := xlMetaV2{}
-	return xlMeta, xlMeta.AddVersion(fi)
-}
-
 func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) {
 	versionID := ""
 	var uv uuid.UUID
@@ -937,7 +954,7 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 	versionID := ""
 	var uv uuid.UUID
 	// check if the version is not "null"
-	if !bytes.Equal(j.VersionID[:], uv[:]) {
+	if j.VersionID != uv {
 		versionID = uuid.UUID(j.VersionID).String()
 	}
 	fi := FileInfo{
@@ -1233,7 +1250,7 @@ func (z xlMetaV2) ListVersions(volume, path string) ([]FileInfo, time.Time, erro
 		versions = append(versions, fi)
 	}
 
-	sort.Sort(versionsSorter(versions))
+	versionsSorter(versions).sort()
 
 	for i := range versions {
 		versions[i].NumVersions = len(versions)

cmd/xl-storage.go

@@ -852,11 +852,10 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		if versionID == "" {
 			versionID = nullVersionID
 		}
-		xlMeta.data.remove(versionID)
 		// PR #11758 used DataDir, preserve it
 		// for users who might have used master
 		// branch
-		xlMeta.data.remove(dataDir)
+		xlMeta.data.remove(versionID, dataDir)
 		filePath := pathJoin(volumeDir, path, dataDir)
 		if err = checkPathLength(filePath); err != nil {
 			return err
@@ -940,7 +939,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 
 	var xlMeta xlMetaV2
 	if !isXL2V1Format(buf) {
-		xlMeta, err = newXLMetaV2(fi)
+		err = xlMeta.AddVersion(fi)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return err