diff --git a/cmd/gateway/gcs/gateway-gcs_test.go b/cmd/gateway/gcs/gateway-gcs_test.go index 7b861768d..6745cf171 100644 --- a/cmd/gateway/gcs/gateway-gcs_test.go +++ b/cmd/gateway/gcs/gateway-gcs_test.go @@ -190,13 +190,12 @@ func TestFromMinioClientListBucketResultToV2Info(t *testing.T) { // Test for gcsParseProjectID func TestGCSParseProjectID(t *testing.T) { - f, err := ioutil.TempFile("", "") + f, err := ioutil.TempFile("", "TestGCSParseProjectID-*") if err != nil { t.Error(err) return } defer os.Remove(f.Name()) - defer f.Close() contents := ` { @@ -205,6 +204,7 @@ func TestGCSParseProjectID(t *testing.T) { } ` f.WriteString(contents) + f.Close() projectID, err := gcsParseProjectID(f.Name()) if err != nil { t.Fatal(err) @@ -217,8 +217,20 @@ func TestGCSParseProjectID(t *testing.T) { t.Errorf(`Expected to fail but succeeded reading "non-existent"`) } - f.WriteString(`,}`) - + contents = ` +{ + "type": "service_account", + "project_id": "miniotesting" +},} +` + f, err = ioutil.TempFile("", "TestGCSParseProjectID-*") + if err != nil { + t.Error(err) + return + } + defer os.Remove(f.Name()) + f.WriteString(contents) + f.Close() if _, err := gcsParseProjectID(f.Name()); err == nil { t.Errorf(`Expected to fail reading corrupted credentials file`) } diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go index a423d8623..049306925 100644 --- a/cmd/storage-rest-server.go +++ b/cmd/storage-rest-server.go @@ -852,6 +852,8 @@ func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse { // The returned reader contains the payload and must be closed if no error is returned. func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error { var tmp [1]byte + // 8K copy buffer, reused for less allocs... + var buf [8 << 10]byte for { _, err := io.ReadFull(respBody, tmp[:]) if err != nil { @@ -861,7 +863,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error { switch tmp[0] { case 0: // 0 is unbuffered, copy the rest. - _, err := io.Copy(w, respBody) + _, err := io.CopyBuffer(w, respBody, buf[:]) if err == io.EOF { return nil } @@ -880,7 +882,7 @@ func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error { return err } length := binary.LittleEndian.Uint32(tmp[:]) - _, err = io.CopyN(w, respBody, int64(length)) + _, err = io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf[:]) if err != nil { return err } diff --git a/cmd/xl-storage-format-utils.go b/cmd/xl-storage-format-utils.go index 68089c151..b024c0fd8 100644 --- a/cmd/xl-storage-format-utils.go +++ b/cmd/xl-storage-format-utils.go @@ -18,21 +18,24 @@ package cmd import ( + "sort" + jsoniter "github.com/json-iterator/go" ) +// versionsSorter sorts FileInfo slices by version. type versionsSorter []FileInfo -func (v versionsSorter) Len() int { return len(v) } -func (v versionsSorter) Swap(i, j int) { v[i], v[j] = v[j], v[i] } -func (v versionsSorter) Less(i, j int) bool { - if v[i].IsLatest { - return true - } - if v[j].IsLatest { - return false - } - return v[i].ModTime.After(v[j].ModTime) +func (v versionsSorter) sort() { + sort.Slice(v, func(i, j int) bool { + if v[i].IsLatest { + return true + } + if v[j].IsLatest { + return false + } + return v[i].ModTime.After(v[j].ModTime) + }) } func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) { diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 837e3efba..1fbce1fd1 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -524,13 +524,35 @@ func (x *xlMetaInlineData) rename(oldKey, newKey string) bool { return true } -// remove will remove a key. -// Returns whether the key was found. -func (x *xlMetaInlineData) remove(key string) bool { +// remove will remove one or more keys. +// Returns true if any key was found. +func (x *xlMetaInlineData) remove(keys ...string) bool { in := x.afterVersion() sz, buf, _ := msgp.ReadMapHeaderBytes(in) - keys := make([][]byte, 0, sz) - vals := make([][]byte, 0, sz) + newKeys := make([][]byte, 0, sz) + newVals := make([][]byte, 0, sz) + var removeKey func(s []byte) bool + + // Copy if big number of compares... + if len(keys) > 5 && sz > 5 { + mKeys := make(map[string]struct{}, len(keys)) + for _, key := range keys { + mKeys[key] = struct{}{} + } + removeKey = func(s []byte) bool { + _, ok := mKeys[string(s)] + return ok + } + } else { + removeKey = func(s []byte) bool { + for _, key := range keys { + if key == string(s) { + return true + } + } + return false + } + } // Version plus header... plSize := 1 + msgp.MapHeaderSize @@ -546,10 +568,10 @@ func (x *xlMetaInlineData) remove(key string) bool { if err != nil { break } - if string(foundKey) != key { + if !removeKey(foundKey) { plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal) - keys = append(keys, foundKey) - vals = append(vals, foundVal) + newKeys = append(newKeys, foundKey) + newVals = append(newVals, foundVal) } else { found = true } @@ -559,13 +581,13 @@ func (x *xlMetaInlineData) remove(key string) bool { return false } // If none left... - if len(keys) == 0 { + if len(newKeys) == 0 { *x = nil return true } // Reserialize... - x.serialize(plSize, keys, vals) + x.serialize(plSize, newKeys, newVals) return true } @@ -891,11 +913,6 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error { return nil } -func newXLMetaV2(fi FileInfo) (xlMetaV2, error) { - xlMeta := xlMetaV2{} - return xlMeta, xlMeta.AddVersion(fi) -} - func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) { versionID := "" var uv uuid.UUID @@ -937,7 +954,7 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { versionID := "" var uv uuid.UUID // check if the version is not "null" - if !bytes.Equal(j.VersionID[:], uv[:]) { + if j.VersionID != uv { versionID = uuid.UUID(j.VersionID).String() } fi := FileInfo{ @@ -1233,7 +1250,7 @@ func (z xlMetaV2) ListVersions(volume, path string) ([]FileInfo, time.Time, erro versions = append(versions, fi) } - sort.Sort(versionsSorter(versions)) + versionsSorter(versions).sort() for i := range versions { versions[i].NumVersions = len(versions) diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 3ad7580cd..3ec61c480 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -852,11 +852,10 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F if versionID == "" { versionID = nullVersionID } - xlMeta.data.remove(versionID) // PR #11758 used DataDir, preserve it // for users who might have used master // branch - xlMeta.data.remove(dataDir) + xlMeta.data.remove(versionID, dataDir) filePath := pathJoin(volumeDir, path, dataDir) if err = checkPathLength(filePath); err != nil { return err @@ -940,7 +939,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F var xlMeta xlMetaV2 if !isXL2V1Format(buf) { - xlMeta, err = newXLMetaV2(fi) + err = xlMeta.AddVersion(fi) if err != nil { logger.LogIf(ctx, err) return err