fix: simplify data structure before release (#9968)

- additionally upgrade to msgp@v1.1.2
- change StatModTime,StatSize fields as
  simple Size/ModTime
- reduce 50000 entries per List batch to 10000
  as client needs to wait too long to see the
  first batch some times which is not desired
  and it is worth we write the data as soon
  as we have it.

cmd/api-response.go

@@ -35,7 +35,7 @@ import (
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 50000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.

cmd/xl-storage-format-v2.go

@@ -156,8 +156,8 @@ type xlMetaV2Object struct {
 	PartETags          []string          `json:"PartETags" msg:"PartETags"`                       // Part ETags
 	PartSizes          []int64           `json:"PartSizes" msg:"PartSizes"`                       // Part Sizes
 	PartActualSizes    []int64           `json:"PartASizes,omitempty" msg:"PartASizes,omitempty"` // Part ActualSizes (compression)
-	StatSize           int64             `json:"Size" msg:"Size"`                                 // Object version size
+	Size               int64             `json:"Size" msg:"Size"`                                 // Object version size
-	StatModTime        int64             `json:"MTime" msg:"MTime"`                               // Object version modified time
+	ModTime            int64             `json:"MTime" msg:"MTime"`                               // Object version modified time
 	MetaSys            map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"`       // Object version internal metadata
 	MetaUser           map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,omitempty"`       // Object version metadata set by user
 }
@@ -184,7 +184,7 @@ func (j xlMetaV2Version) Valid() bool {
 			j.ObjectV2.ErasureAlgorithm.valid() &&
 			j.ObjectV2.BitrotChecksumAlgo.valid() &&
 			isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) &&
-			j.ObjectV2.StatModTime > 0
+			j.ObjectV2.ModTime > 0
 	case DeleteType:
 		return j.DeleteMarker != nil &&
 			j.DeleteMarker.ModTime > 0
@@ -271,8 +271,8 @@ func (z *xlMetaV2) AddVersion(fi FileInfo) error {
 		ObjectV2: &xlMetaV2Object{
 			VersionID:          uv,
 			DataDir:            dd,
-			StatSize:           fi.Size,
+			Size:               fi.Size,
-			StatModTime:        fi.ModTime.UnixNano(),
+			ModTime:            fi.ModTime.UnixNano(),
 			ErasureAlgorithm:   ReedSolomon,
 			ErasureM:           fi.Erasure.DataBlocks,
 			ErasureN:           fi.Erasure.ParityBlocks,
@@ -373,8 +373,8 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 	fi := FileInfo{
 		Volume:    volume,
 		Name:      path,
-		Size:      j.StatSize,
+		Size:      j.Size,
-		ModTime:   time.Unix(0, j.StatModTime).UTC(),
+		ModTime:   time.Unix(0, j.ModTime).UTC(),
 		VersionID: versionID,
 	}
 	fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers))
@@ -463,7 +463,7 @@ func (z xlMetaV2) TotalSize() int64 {
 	for i := range z.Versions {
 		switch z.Versions[i].Type {
 		case ObjectType:
-			total += z.Versions[i].ObjectV2.StatSize
+			total += z.Versions[i].ObjectV2.Size
 		case LegacyType:
 			total += z.Versions[i].ObjectV1.Stat.Size
 		}
@@ -488,8 +488,6 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
 			fi, err = version.DeleteMarker.ToFileInfo(volume, path)
 		case LegacyType:
 			fi, err = version.ObjectV1.ToFileInfo(volume, path)
-		default:
-			continue
 		}
 		if err != nil {
 			return nil, latestModTime, err
@@ -498,12 +496,7 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
 			latestModTime = fi.ModTime
 			latestVersionID = fi.VersionID
 		}
-		switch version.Type {
+		versions = append(versions, fi)
-		case LegacyType:
-			fallthrough
-		case ObjectType, DeleteType:
-			versions = append(versions, fi)
-		}
 	}
 
 	// We didn't find the version in delete markers so latest version
@@ -521,44 +514,53 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim
 
 // ToFileInfo converts xlMetaV2 into a common FileInfo datastructure
 // for consumption across callers.
-func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) {
+func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (fi FileInfo, err error) {
 	var uv uuid.UUID
 	if versionID != "" {
 		uv, _ = uuid.Parse(versionID)
 	}
 
+	var latestModTime time.Time
+	var latestIndex int
+	for i, version := range z.Versions {
+		if !version.Valid() {
+			logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
+			return FileInfo{}, errFileNotFound
+		}
+		var modTime time.Time
+		switch version.Type {
+		case ObjectType:
+			modTime = time.Unix(0, version.ObjectV2.ModTime)
+		case DeleteType:
+			modTime = time.Unix(0, version.DeleteMarker.ModTime)
+		case LegacyType:
+			modTime = version.ObjectV1.Stat.ModTime
+		}
+		if modTime.After(latestModTime) {
+			latestModTime = modTime
+			latestIndex = i
+		}
+	}
+
 	if versionID == "" {
-		var latestModTime time.Time
+		if len(z.Versions) >= 1 {
-		var latestIndex int
+			if !z.Versions[latestIndex].Valid() {
-		for i, version := range z.Versions {
+				logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", z.Versions[latestIndex]))
-			if !version.Valid() {
-				logger.LogIf(GlobalContext, fmt.Errorf("invalid version detected %#v", version))
 				return FileInfo{}, errFileNotFound
 			}
-			var modTime time.Time
-			switch version.Type {
-			case ObjectType:
-				modTime = time.Unix(0, version.ObjectV2.StatModTime)
-			case DeleteType:
-				modTime = time.Unix(0, version.DeleteMarker.ModTime)
-			case LegacyType:
-				modTime = version.ObjectV1.Stat.ModTime
-			default:
-				continue
-			}
-			if modTime.After(latestModTime) {
-				latestModTime = modTime
-				latestIndex = i
-			}
-		}
-		if len(z.Versions) >= 1 {
 			switch z.Versions[latestIndex].Type {
 			case ObjectType:
-				return z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path)
+				fi, err = z.Versions[latestIndex].ObjectV2.ToFileInfo(volume, path)
+				fi.IsLatest = true
+				return fi, err
 			case DeleteType:
-				return z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path)
+				fi, err = z.Versions[latestIndex].DeleteMarker.ToFileInfo(volume, path)
+				fi.IsLatest = true
+				return fi, err
 			case LegacyType:
-				return z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path)
+				fi, err = z.Versions[latestIndex].ObjectV1.ToFileInfo(volume, path)
+				fi.IsLatest = true
+				return fi, err
 			}
 		}
 		return FileInfo{}, errFileNotFound
@@ -575,23 +577,22 @@ func (z xlMetaV2) ToFileInfo(volume, path, versionID string) (FileInfo, error) {
 		switch version.Type {
 		case ObjectType:
 			if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) {
-				return version.ObjectV2.ToFileInfo(volume, path)
+				fi, err = version.ObjectV2.ToFileInfo(volume, path)
+				fi.IsLatest = latestModTime.Equal(fi.ModTime)
+				return fi, err
 			}
 		case LegacyType:
 			if version.ObjectV1.VersionID == versionID {
-				return version.ObjectV1.ToFileInfo(volume, path)
+				fi, err = version.ObjectV1.ToFileInfo(volume, path)
+				fi.IsLatest = latestModTime.Equal(fi.ModTime)
+				return fi, err
 			}
 		case DeleteType:
 			if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) {
-				return version.DeleteMarker.ToFileInfo(volume, path)
+				fi, err = version.DeleteMarker.ToFileInfo(volume, path)
+				fi.IsLatest = latestModTime.Equal(fi.ModTime)
+				return fi, err
 			}
-		default:
-			logger.LogIf(GlobalContext, fmt.Errorf("unknown version type: %v", version.Type))
-			if versionID == "" {
-				return FileInfo{}, errFileNotFound
-			}
-
-			return FileInfo{}, errFileVersionNotFound
 		}
 	}
 

cmd/xl-storage-format-v2_gen.go

@@ -606,15 +606,15 @@ func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 			}
 		case "Size":
-			z.StatSize, err = dc.ReadInt64()
+			z.Size, err = dc.ReadInt64()
 			if err != nil {
-				err = msgp.WrapError(err, "StatSize")
+				err = msgp.WrapError(err, "Size")
 				return
 			}
 		case "MTime":
-			z.StatModTime, err = dc.ReadInt64()
+			z.ModTime, err = dc.ReadInt64()
 			if err != nil {
-				err = msgp.WrapError(err, "StatModTime")
+				err = msgp.WrapError(err, "ModTime")
 				return
 			}
 		case "MetaSys":
@@ -885,9 +885,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteInt64(z.StatSize)
+	err = en.WriteInt64(z.Size)
 	if err != nil {
-		err = msgp.WrapError(err, "StatSize")
+		err = msgp.WrapError(err, "Size")
 		return
 	}
 	// write "MTime"
@@ -895,9 +895,9 @@ func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteInt64(z.StatModTime)
+	err = en.WriteInt64(z.ModTime)
 	if err != nil {
-		err = msgp.WrapError(err, "StatModTime")
+		err = msgp.WrapError(err, "ModTime")
 		return
 	}
 	if (zb0001Mask & 0x8000) == 0 { // if not empty
@@ -1032,10 +1032,10 @@ func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// string "Size"
 	o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendInt64(o, z.StatSize)
+	o = msgp.AppendInt64(o, z.Size)
 	// string "MTime"
 	o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65)
-	o = msgp.AppendInt64(o, z.StatModTime)
+	o = msgp.AppendInt64(o, z.ModTime)
 	if (zb0001Mask & 0x8000) == 0 { // if not empty
 		// string "MetaSys"
 		o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73)
@@ -1227,15 +1227,15 @@ func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 			}
 		case "Size":
-			z.StatSize, bts, err = msgp.ReadInt64Bytes(bts)
+			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "StatSize")
+				err = msgp.WrapError(err, "Size")
 				return
 			}
 		case "MTime":
-			z.StatModTime, bts, err = msgp.ReadInt64Bytes(bts)
+			z.ModTime, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
-				err = msgp.WrapError(err, "StatModTime")
+				err = msgp.WrapError(err, "ModTime")
 				return
 			}
 		case "MetaSys":

go.mod

@@ -106,7 +106,7 @@ require (
 	github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94
 	github.com/tidwall/gjson v1.3.5
 	github.com/tidwall/sjson v1.0.4
-	github.com/tinylib/msgp v1.1.1
+	github.com/tinylib/msgp v1.1.2
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/ugorji/go v1.1.5-pre // indirect
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a

go.sum

@@ -419,8 +419,8 @@ github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tidwall/sjson v1.0.4 h1:UcdIRXff12Lpnu3OLtZvnc03g4vH2suXDXhBwBqmzYg=
 github.com/tidwall/sjson v1.0.4/go.mod h1:bURseu1nuBkFpIES5cz6zBtjmYeOQmEESshn7VpF15Y=
-github.com/tinylib/msgp v1.1.1 h1:TnCZ3FIuKeaIy+F45+Cnp+caqdXGy4z74HvwXN+570Y=
+github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
-github.com/tinylib/msgp v1.1.1/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=