self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-10-02 22:24:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

admin-info: add DeleteMarkers count (#17659)

Browse source
This commit is contained in:
Krishnan Parthasarathi 2023-07-18 10:49:40 -07:00 committed by GitHub
parent 49638fa533
commit 0120ff93bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 80 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
cmd/admin-handlers.go
View file

@ -1806,6 +1806,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
dataUsageInfo := cache.dui(dataUsageRoot, nil)
erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
erasureSet.DeleteMarkersCount = dataUsageInfo.DeleteMarkersTotalCount
erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
}
}
@ -1855,6 +1856,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
buckets := madmin.Buckets{}
objects := madmin.Objects{}
versions := madmin.Versions{}
deleteMarkers := madmin.DeleteMarkers{}
usage := madmin.Usage{}
objectAPI := newObjectLayerFn()
@ -1867,10 +1869,12 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount}
deleteMarkers = madmin.DeleteMarkers{Count: dataUsageInfo.DeleteMarkersTotalCount}
usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
} else {
buckets = madmin.Buckets{Error: err.Error()}
objects = madmin.Objects{Error: err.Error()}
deleteMarkers = madmin.DeleteMarkers{Error: err.Error()}
usage = madmin.Usage{Error: err.Error()}
}
@ -1909,19 +1913,20 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
}
return madmin.InfoMessage{
Mode: string(mode),
Domain: domain,
Region: globalSite.Region,
SQSARN: globalEventNotifier.GetARNList(false),
DeploymentID: globalDeploymentID,
Buckets: buckets,
Objects: objects,
Versions: versions,
Usage: usage,
Services: services,
Backend: backend,
Servers: servers,
Pools: poolsInfo,
Mode: string(mode),
Domain: domain,
Region: globalSite.Region,
SQSARN: globalEventNotifier.GetARNList(false),
DeploymentID: globalDeploymentID,
Buckets: buckets,
Objects: objects,
Versions: versions,
DeleteMarkers: deleteMarkers,
Usage: usage,
Services: services,
Backend: backend,
Servers: servers,
Pools: poolsInfo,
}
}

1
cmd/data-scanner.go
View file

@ -845,6 +845,7 @@ type scannerItem struct {
type sizeSummary struct {
totalSize int64
versions uint64
deleteMarkers uint64
replicatedSize int64
pendingSize int64
failedSize int64

19
cmd/data-usage-cache.go
View file

@ -57,6 +57,7 @@ type dataUsageEntry struct {
Size int64 `msg:"sz"`
Objects uint64 `msg:"os"`
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
DeleteMarkers uint64 `msg:"dms"`
ObjSizes sizeHistogram `msg:"szs"`
ObjVersions versionsHistogram `msg:"vh"`
ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
@ -328,6 +329,7 @@ type dataUsageCacheInfo struct {
func (e *dataUsageEntry) addSizes(summary sizeSummary) {
e.Size += summary.totalSize
e.Versions += summary.versions
e.DeleteMarkers += summary.deleteMarkers
e.ObjSizes.add(summary.totalSize)
e.ObjVersions.add(summary.versions)
@ -366,6 +368,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
func (e *dataUsageEntry) merge(other dataUsageEntry) {
e.Objects += other.Objects
e.Versions += other.Versions
e.DeleteMarkers += other.DeleteMarkers
e.Size += other.Size
if other.ReplicationStats != nil {
if e.ReplicationStats == nil {
@ -531,13 +534,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
}
flat := d.flatten(*e)
dui := DataUsageInfo{
LastUpdate: d.Info.LastUpdate,
ObjectsTotalCount: flat.Objects,
VersionsTotalCount: flat.Versions,
ObjectsTotalSize: uint64(flat.Size),
BucketsCount: uint64(len(e.Children)),
BucketsUsage: d.bucketsUsageInfo(buckets),
TierStats: d.tiersUsageInfo(buckets),
LastUpdate: d.Info.LastUpdate,
ObjectsTotalCount: flat.Objects,
VersionsTotalCount: flat.Versions,
DeleteMarkersTotalCount: flat.DeleteMarkers,
ObjectsTotalSize: uint64(flat.Size),
BucketsCount: uint64(len(e.Children)),
BucketsUsage: d.bucketsUsageInfo(buckets),
TierStats: d.tiersUsageInfo(buckets),
}
return dui
}
@ -805,6 +809,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
Size: uint64(flat.Size),
VersionsCount: flat.Versions,
ObjectsCount: flat.Objects,
DeleteMarkersCount: flat.DeleteMarkers,
ObjectSizesHistogram: flat.ObjSizes.toMap(),
ObjectVersionsHistogram: flat.ObjVersions.toMap(),
}

51
cmd/data-usage-cache_gen.go
View file

@ -1522,6 +1522,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
err = msgp.WrapError(err, "Versions")
return
}
case "dms":
z.DeleteMarkers, err = dc.ReadUint64()
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
case "szs":
var zb0002 uint32
zb0002, err = dc.ReadArrayHeader()
@ -1614,16 +1620,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
// EncodeMsg implements msgp.Encodable
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
// omitempty: check for empty values
zb0001Len := uint32(9)
var zb0001Mask uint16 /* 9 bits */
zb0001Len := uint32(10)
var zb0001Mask uint16 /* 10 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x40
zb0001Mask |= 0x80
}
if z.AllTierStats == nil {
zb0001Len--
zb0001Mask |= 0x80
zb0001Mask |= 0x100
}
// variable map header, size zb0001Len
err = en.Append(0x80 | uint8(zb0001Len))
@ -1673,6 +1679,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
err = msgp.WrapError(err, "Versions")
return
}
// write "dms"
err = en.Append(0xa3, 0x64, 0x6d, 0x73)
if err != nil {
return
}
err = en.WriteUint64(z.DeleteMarkers)
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
// write "szs"
err = en.Append(0xa3, 0x73, 0x7a, 0x73)
if err != nil {
@ -1707,7 +1723,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
return
}
}
if (zb0001Mask & 0x40) == 0 { // if not empty
if (zb0001Mask & 0x80) == 0 { // if not empty
// write "rs"
err = en.Append(0xa2, 0x72, 0x73)
if err != nil {
@ -1726,7 +1742,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
}
}
}
if (zb0001Mask & 0x80) == 0 { // if not empty
if (zb0001Mask & 0x100) == 0 { // if not empty
// write "ats"
err = en.Append(0xa3, 0x61, 0x74, 0x73)
if err != nil {
@ -1762,16 +1778,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
o = msgp.Require(b, z.Msgsize())
// omitempty: check for empty values
zb0001Len := uint32(9)
var zb0001Mask uint16 /* 9 bits */
zb0001Len := uint32(10)
var zb0001Mask uint16 /* 10 bits */
_ = zb0001Mask
if z.ReplicationStats == nil {
zb0001Len--
zb0001Mask |= 0x40
zb0001Mask |= 0x80
}
if z.AllTierStats == nil {
zb0001Len--
zb0001Mask |= 0x80
zb0001Mask |= 0x100
}
// variable map header, size zb0001Len
o = append(o, 0x80|uint8(zb0001Len))
@ -1794,6 +1810,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
// string "vs"
o = append(o, 0xa2, 0x76, 0x73)
o = msgp.AppendUint64(o, z.Versions)
// string "dms"
o = append(o, 0xa3, 0x64, 0x6d, 0x73)
o = msgp.AppendUint64(o, z.DeleteMarkers)
// string "szs"
o = append(o, 0xa3, 0x73, 0x7a, 0x73)
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
@ -1806,7 +1825,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
for za0002 := range z.ObjVersions {
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
}
if (zb0001Mask & 0x40) == 0 { // if not empty
if (zb0001Mask & 0x80) == 0 { // if not empty
// string "rs"
o = append(o, 0xa2, 0x72, 0x73)
if z.ReplicationStats == nil {
@ -1819,7 +1838,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
}
}
}
if (zb0001Mask & 0x80) == 0 { // if not empty
if (zb0001Mask & 0x100) == 0 { // if not empty
// string "ats"
o = append(o, 0xa3, 0x61, 0x74, 0x73)
if z.AllTierStats == nil {
@ -1880,6 +1899,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
err = msgp.WrapError(err, "Versions")
return
}
case "dms":
z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
if err != nil {
err = msgp.WrapError(err, "DeleteMarkers")
return
}
case "szs":
var zb0002 uint32
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@ -1970,7 +1995,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
func (z *dataUsageEntry) Msgsize() (s int) {
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
if z.ReplicationStats == nil {
s += msgp.NilSize
} else {

6
cmd/data-usage-utils.go
View file

@ -62,6 +62,7 @@ type BucketUsageInfo struct {
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
VersionsCount uint64 `json:"versionsCount"`
DeleteMarkersCount uint64 `json:"deleteMarkersCount"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
}
@ -75,9 +76,12 @@ type DataUsageInfo struct {
// Objects total count across all buckets
ObjectsTotalCount uint64 `json:"objectsCount"`
// Objects total count across all buckets
// Versions total count across all buckets
VersionsTotalCount uint64 `json:"versionsCount"`
// Delete markers total count across all buckets
DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"`
// Objects total size across all buckets
ObjectsTotalSize uint64 `json:"objectsTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`

3
cmd/xl-storage.go
View file

@ -547,6 +547,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
done = globalScannerMetrics.time(scannerMetricApplyVersion)
sz := item.applyActions(ctx, objAPI, oi, &sizeS)
done()
if oi.DeleteMarker {
sizeS.deleteMarkers++
}
if oi.VersionID != "" && sz == oi.Size {
sizeS.versions++
}

2
go.mod
View file

@ -49,7 +49,7 @@ require (
github.com/minio/dperf v0.5.0
github.com/minio/highwayhash v1.0.2
github.com/minio/kes-go v0.1.0
github.com/minio/madmin-go/v3 v3.0.7
github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25
github.com/minio/minio-go/v7 v7.0.61
github.com/minio/mux v1.9.0
github.com/minio/pkg v1.7.5

4
go.sum
View file

@ -484,8 +484,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc=
github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo=
github.com/minio/madmin-go/v3 v3.0.7 h1:nuRwrqarFrkzbUiA36H/HKAcuNr8J9TjKlWRlua7lNo=
github.com/minio/madmin-go/v3 v3.0.7/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk=
github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25 h1:4zQOJq6tbY6MDXbdPw5tuajM9rQroJ7eIRGs5mxr40w=
github.com/minio/madmin-go/v3 v3.0.8-0.20230717144230-b697fb223a25/go.mod h1:lPrMoc1aeiIWmmrxBthkDqzMPQwC/Lu9ByuyM2wenJk=
github.com/minio/mc v0.0.0-20230713161043-15ea1125e377 h1:jcIt177m3jJCrDEAZFhgUcq+PdMnpfsF8QrFNSbG0nU=
github.com/minio/mc v0.0.0-20230713161043-15ea1125e377/go.mod h1:eXUhMFdo6O9d0axon9FVNaEtCDPMJdZW7K7jyYQRR5Q=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=