minio/cmd/data-usage-utils.go
Aditya Manthramurthy b2c5b75efa
feat: Add Metrics V3 API (#19068)
Metrics v3 is mainly a reorganization of metrics into smaller groups of
metrics and the removal of internal aggregation of metrics received from
peer nodes in a MinIO cluster.

This change adds the endpoint `/minio/metrics/v3` as the top-level metrics
endpoint and under this, various sub-endpoints are implemented. These
are currently documented in `docs/metrics/v3.md`

The handler will serve metrics at any path
`/minio/metrics/v3/PATH`, as follows:

when PATH is a sub-endpoint listed above => serves the group of
metrics under that path; or when PATH is a (non-empty) parent 
directory of the sub-endpoints listed above => serves metrics
from each child sub-endpoint of PATH. otherwise, returns a no 
resource found error

All available metrics are listed in the `docs/metrics/v3.md`. More will
be added subsequently.
2024-03-10 01:15:15 -08:00

170 lines
6.4 KiB
Go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"sort"
"time"
"github.com/minio/madmin-go/v3"
)
// BucketTargetUsageInfo - bucket target usage info provides
// - replicated size for all objects sent to this target
// - replica size for all objects received from this target
// - replication pending size for all objects pending replication to this target
// - replication failed size for all objects failed replication to this target
// - replica pending count
// - replica failed count
type BucketTargetUsageInfo struct {
ReplicationPendingSize uint64 `json:"objectsPendingReplicationTotalSize"`
ReplicationFailedSize uint64 `json:"objectsFailedReplicationTotalSize"`
ReplicatedSize uint64 `json:"objectsReplicatedTotalSize"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
ReplicationFailedCount uint64 `json:"objectsFailedReplicationCount"`
ReplicatedCount uint64 `json:"objectsReplicatedCount"`
}
// BucketUsageInfo - bucket usage info provides
// - total size of the bucket
// - total objects in a bucket
// - object size histogram per bucket
type BucketUsageInfo struct {
Size uint64 `json:"size"`
// Following five fields suffixed with V1 are here for backward compatibility
// Total Size for objects that have not yet been replicated
ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
// Total size for objects that have witness one or more failures and will be retried
ReplicationFailedSizeV1 uint64 `json:"objectsFailedReplicationTotalSize"`
// Total size for objects that have been replicated to destination
ReplicatedSizeV1 uint64 `json:"objectsReplicatedTotalSize"`
// Total number of objects pending replication
ReplicationPendingCountV1 uint64 `json:"objectsPendingReplicationCount"`
// Total number of objects that failed replication
ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"`
ObjectsCount uint64 `json:"objectsCount"`
ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"`
ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"`
VersionsCount uint64 `json:"versionsCount"`
DeleteMarkersCount uint64 `json:"deleteMarkersCount"`
ReplicaSize uint64 `json:"objectReplicaTotalSize"`
ReplicaCount uint64 `json:"objectReplicaCount"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
}
// DataUsageInfo represents data usage stats of the underlying Object API
type DataUsageInfo struct {
TotalCapacity uint64 `json:"capacity,omitempty"`
TotalUsedCapacity uint64 `json:"usedCapacity,omitempty"`
TotalFreeCapacity uint64 `json:"freeCapacity,omitempty"`
// LastUpdate is the timestamp of when the data usage info was last updated.
// This does not indicate a full scan.
LastUpdate time.Time `json:"lastUpdate"`
// Objects total count across all buckets
ObjectsTotalCount uint64 `json:"objectsCount"`
// Versions total count across all buckets
VersionsTotalCount uint64 `json:"versionsCount"`
// Delete markers total count across all buckets
DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"`
// Objects total size across all buckets
ObjectsTotalSize uint64 `json:"objectsTotalSize"`
ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
// Total number of buckets in this cluster
BucketsCount uint64 `json:"bucketsCount"`
// Buckets usage info provides following information across all buckets
// - total size of the bucket
// - total objects in a bucket
// - object size histogram per bucket
BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"`
// Deprecated kept here for backward compatibility reasons.
BucketSizes map[string]uint64 `json:"bucketsSizes"`
// TierStats contains per-tier stats of all configured remote tiers
TierStats *allTierStats `json:"tierStats,omitempty"`
}
func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
if dui.TierStats == nil {
return nil
}
if globalTierConfigMgr.Empty() {
return nil
}
ts := make(map[string]madmin.TierStats)
dui.TierStats.populateStats(ts)
infos := make([]madmin.TierInfo, 0, len(ts))
for tier, stats := range ts {
infos = append(infos, madmin.TierInfo{
Name: tier,
Type: globalTierConfigMgr.TierType(tier),
Stats: stats,
})
}
sort.Slice(infos, func(i, j int) bool {
if infos[i].Type == "internal" {
return true
}
if infos[j].Type == "internal" {
return false
}
return infos[i].Name < infos[j].Name
})
return infos
}
func (dui DataUsageInfo) tierMetrics() (metrics []MetricV2) {
if dui.TierStats == nil {
return nil
}
// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
// minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
// minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
for tier, st := range dui.TierStats.Tiers {
metrics = append(metrics, MetricV2{
Description: getClusterTransitionedBytesMD(),
Value: float64(st.TotalSize),
VariableLabels: map[string]string{"tier": tier},
})
metrics = append(metrics, MetricV2{
Description: getClusterTransitionedObjectsMD(),
Value: float64(st.NumObjects),
VariableLabels: map[string]string{"tier": tier},
})
metrics = append(metrics, MetricV2{
Description: getClusterTransitionedVersionsMD(),
Value: float64(st.NumVersions),
VariableLabels: map[string]string{"tier": tier},
})
}
return metrics
}