Reuse buffers when writing metadata (#13040)

Simplify returning buffers.

Tested using `warp mixed --duration=1m --obj.size=100K`:

```
Operation: DELETE
Operations: 7148 -> 7642
* Average: +6.77% (+8.1) obj/s
-------------------
Operation: GET
Operations: 32200 -> 34403
* Average: +6.74% (+3.5 MiB/s) throughput, +6.74% (+36.2) obj/s
* First Byte: Average: -105.403µs (-3%), Median: -309µs (-11%), Best: -2.7µs (-0%), Worst: +3.5637ms (+3%)
-------------------
Operation: PUT
Operations: 10741 -> 11475
* Average: +6.78% (+1.2 MiB/s) throughput, +6.78% (+12.1) obj/s
-------------------
Operation: STAT
Operations: 21465 -> 22927
* Average: +6.71% (+24.0) obj/s
```

cmd/metacache-entries.go

@@ -497,9 +497,7 @@ func (m *metaCacheEntriesSorted) forwardTo(s string) {
 	})
 	if m.reuse {
 		for i, entry := range m.o[:idx] {
-			if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
-				metaDataPool.Put(entry.metadata)
-			}
+			metaDataPoolPut(entry.metadata)
 			m.o[i].metadata = nil
 		}
 	}
@@ -517,9 +515,7 @@ func (m *metaCacheEntriesSorted) forwardPast(s string) {
 	})
 	if m.reuse {
 		for i, entry := range m.o[:idx] {
-			if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
-				metaDataPool.Put(entry.metadata)
-			}
+			metaDataPoolPut(entry.metadata)
 			m.o[i].metadata = nil
 		}
 	}
@@ -739,9 +735,7 @@ func (m *metaCacheEntriesSorted) truncate(n int) {
 	if len(m.o) > n {
 		if m.reuse {
 			for i, entry := range m.o[n:] {
-				if len(entry.metadata) >= metaDataReadDefault && len(entry.metadata) < metaDataReadDefault*4 {
-					metaDataPool.Put(entry.metadata)
-				}
+				metaDataPoolPut(entry.metadata)
 				m.o[n+i].metadata = nil
 			}
 		}

cmd/metacache-stream.go

@@ -143,9 +143,7 @@ func (w *metacacheWriter) write(objs ...metaCacheEntry) error {
 			return err
 		}
 		if w.reuseBlocks || o.reusable {
-			if cap(o.metadata) >= metaDataReadDefault && cap(o.metadata) < metaDataReadDefault*4 {
-				metaDataPool.Put(o.metadata)
-			}
+			metaDataPoolPut(o.metadata)
 		}
 	}
 
@@ -360,12 +358,12 @@ func (r *metacacheReader) next() (metaCacheEntry, error) {
 		r.err = err
 		return m, err
 	}
-	m.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0])
+	m.metadata, err = r.mr.ReadBytes(metaDataPoolGet())
 	if err == io.EOF {
 		err = io.ErrUnexpectedEOF
 	}
 	if len(m.metadata) == 0 && cap(m.metadata) >= metaDataReadDefault {
-		metaDataPool.Put(m.metadata)
+		metaDataPoolPut(m.metadata)
 		m.metadata = nil
 	}
 	r.err = err
@@ -520,15 +518,15 @@ func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string
 			r.mr.R.Skip(1)
 			return metaCacheEntriesSorted{o: res}, io.EOF
 		}
-		if meta.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0]); err != nil {
+		if meta.metadata, err = r.mr.ReadBytes(metaDataPoolGet()); err != nil {
 			if err == io.EOF {
 				err = io.ErrUnexpectedEOF
 			}
 			r.err = err
 			return metaCacheEntriesSorted{o: res}, err
 		}
-		if len(meta.metadata) == 0 && cap(meta.metadata) >= metaDataReadDefault {
-			metaDataPool.Put(meta.metadata)
+		if len(meta.metadata) == 0 {
+			metaDataPoolPut(meta.metadata)
 			meta.metadata = nil
 		}
 		if !inclDirs && meta.isDir() {
@@ -579,15 +577,15 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
 			r.err = err
 			return err
 		}
-		if meta.metadata, err = r.mr.ReadBytes(metaDataPool.Get().([]byte)[:0]); err != nil {
+		if meta.metadata, err = r.mr.ReadBytes(metaDataPoolGet()); err != nil {
 			if err == io.EOF {
 				err = io.ErrUnexpectedEOF
 			}
 			r.err = err
 			return err
 		}
-		if len(meta.metadata) == 0 && cap(meta.metadata) >= metaDataReadDefault {
-			metaDataPool.Put(meta.metadata)
+		if len(meta.metadata) == 0 {
+			metaDataPoolPut(meta.metadata)
 			meta.metadata = nil
 		}
 		select {

cmd/xl-storage-format-v2.go

@@ -1437,6 +1437,19 @@ const metaDataReadDefault = 4 << 10
 // Return used metadata byte slices here.
 var metaDataPool = sync.Pool{New: func() interface{} { return make([]byte, 0, metaDataReadDefault) }}
 
+// metaDataPoolGet will return a byte slice with capacity at least metaDataReadDefault.
+// It will be length 0.
+func metaDataPoolGet() []byte {
+	return metaDataPool.Get().([]byte)[:0]
+}
+
+// metaDataPoolPut will put an unused small buffer back into the pool.
+func metaDataPoolPut(buf []byte) {
+	if cap(buf) >= metaDataReadDefault && cap(buf) < metaDataReadDefault*4 {
+		metaDataPool.Put(buf)
+	}
+}
+
 // readXLMetaNoData will load the metadata, but skip data segments.
 // This should only be used when data is never interesting.
 // If data is not xlv2, it is returned in full.
@@ -1448,9 +1461,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		hasFull = false
 	}
 
-	buf := metaDataPool.Get().([]byte)
-	buf = buf[:initial]
-
+	buf := metaDataPoolGet()[:initial]
 	_, err := io.ReadFull(r, buf)
 	if err != nil {
 		return nil, fmt.Errorf("readXLMetaNoData.ReadFull: %w", err)

cmd/xl-storage.go

@@ -903,7 +903,8 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		}
 	}
 	if !lastVersion {
-		buf, err = xlMeta.AppendTo(nil)
+		buf, err = xlMeta.AppendTo(metaDataPoolGet())
+		defer metaDataPoolPut(buf)
 		if err != nil {
 			return err
 		}
@@ -970,7 +971,8 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 			logger.LogIf(ctx, err)
 			return err
 		}
-		buf, err := xlMeta.AppendTo(nil)
+		buf, err := xlMeta.AppendTo(metaDataPoolGet())
+		defer metaDataPoolPut(buf)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return err
@@ -986,6 +988,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 	if err != nil && err != errFileNotFound {
 		return err
 	}
+	defer metaDataPoolPut(buf)
 
 	var xlMeta xlMetaV2
 	if !isXL2V1Format(buf) {
@@ -995,7 +998,8 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 			return err
 		}
 
-		buf, err = xlMeta.AppendTo(nil)
+		buf, err = xlMeta.AppendTo(metaDataPoolGet())
+		defer metaDataPoolPut(buf)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return err
@@ -1011,7 +1015,8 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 			return err
 		}
 
-		buf, err = xlMeta.AppendTo(nil)
+		buf, err = xlMeta.AppendTo(metaDataPoolGet())
+		defer metaDataPoolPut(buf)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return err
@@ -1132,11 +1137,9 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 		return fi, err
 	}
 
-	if len(fi.Data) == 0 && cap(buf) >= metaDataReadDefault && cap(buf) < metaDataReadDefault*4 {
+	if len(fi.Data) == 0 {
 		// We did not read inline data, so we have no references.
-		defer func(b []byte) {
-			metaDataPool.Put(buf)
-		}(buf)
+		defer metaDataPoolPut(buf)
 	}
 
 	if readData {