From 8082d1fed6b5262cb34f74be14bafd462d088160 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 14 Jun 2022 15:14:24 -0700 Subject: [PATCH] add bucket level S3 received/sent bytes (#15084) adds bucket level metrics for bytes received and sent bytes on all S3 API calls. --- cmd/admin-handlers.go | 2 + cmd/generic-handlers.go | 36 +++++++++- cmd/globals.go | 3 + cmd/http-stats.go | 124 +++++++++++++++++++++++++++++--- cmd/metrics-v2.go | 122 ++++++++++++++++++++----------- cmd/notification.go | 1 + cmd/peer-rest-server.go | 2 + docs/metrics/prometheus/list.md | 2 + 8 files changed, 236 insertions(+), 56 deletions(-) diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go index e4f087493..82ddfa5ab 100644 --- a/cmd/admin-handlers.go +++ b/cmd/admin-handlers.go @@ -275,6 +275,8 @@ type ServerConnStats struct { Throughput uint64 `json:"throughput,omitempty"` S3InputBytes uint64 `json:"transferredS3"` S3OutputBytes uint64 `json:"receivedS3"` + AdminInputBytes uint64 `json:"transferredAdmin"` + AdminOutputBytes uint64 `json:"receivedAdmin"` } // ServerHTTPAPIStats holds total number of HTTP operations from/to the server, diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go index 99f747c96..6ce7258e3 100644 --- a/cmd/generic-handlers.go +++ b/cmd/generic-handlers.go @@ -271,6 +271,22 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) { return time.Time{}, ErrMissingDateHeader } +// splitStr splits a string into n parts, empty strings are added +// if we are not able to reach n elements +func splitStr(path, sep string, n int) []string { + splits := strings.SplitN(path, sep, n) + // Add empty strings if we found elements less than nr + for i := n - len(splits); i > 0; i-- { + splits = append(splits, "") + } + return splits +} + +func url2Bucket(p string) (bucket string) { + tokens := splitStr(p, SlashSeparator, 3) + return tokens[1] +} + // setHttpStatsHandler sets a http Stats handler to gather HTTP statistics func setHTTPStatsHandler(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -287,9 +303,23 @@ func setHTTPStatsHandler(h http.Handler) http.Handler { strings.HasPrefix(r.URL.Path, lockRESTPrefix) { globalConnStats.incInputBytes(meteredRequest.BytesRead()) globalConnStats.incOutputBytes(meteredResponse.BytesWritten()) - } else { - globalConnStats.incS3InputBytes(meteredRequest.BytesRead()) - globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten()) + return + } + if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) { + globalConnStats.incAdminInputBytes(meteredRequest.BytesRead()) + globalConnStats.incAdminOutputBytes(meteredResponse.BytesWritten()) + return + } + + globalConnStats.incS3InputBytes(meteredRequest.BytesRead()) + globalConnStats.incS3OutputBytes(meteredResponse.BytesWritten()) + + if r.URL != nil { + bucket := url2Bucket(r.URL.Path) + if bucket != "" && bucket != minioReservedBucket { + globalBucketConnStats.incS3InputBytes(bucket, meteredRequest.BytesRead()) + globalBucketConnStats.incS3OutputBytes(bucket, meteredResponse.BytesWritten()) + } } }) } diff --git a/cmd/globals.go b/cmd/globals.go index 8063aa141..3a968ad71 100644 --- a/cmd/globals.go +++ b/cmd/globals.go @@ -255,6 +255,9 @@ var ( // Global HTTP request statisitics globalHTTPStats = newHTTPStats() + // Global bucket network statistics + globalBucketConnStats = newBucketConnStats() + // Time when the server is started globalBootTime = UTCNow() diff --git a/cmd/http-stats.go b/cmd/http-stats.go index 6cae30bd8..5e00231fe 100644 --- a/cmd/http-stats.go +++ b/cmd/http-stats.go @@ -35,19 +35,21 @@ type ConnStats struct { totalOutputBytes uint64 s3InputBytes uint64 s3OutputBytes uint64 + adminInputBytes uint64 + adminOutputBytes uint64 } -// Increase total input bytes +// Increase internode total input bytes func (s *ConnStats) incInputBytes(n int64) { atomic.AddUint64(&s.totalInputBytes, uint64(n)) } -// Increase total output bytes +// Increase internode total output bytes func (s *ConnStats) incOutputBytes(n int64) { atomic.AddUint64(&s.totalOutputBytes, uint64(n)) } -// Return total input bytes +// Return internode total input bytes func (s *ConnStats) getTotalInputBytes() uint64 { return atomic.LoadUint64(&s.totalInputBytes) } @@ -57,33 +59,55 @@ func (s *ConnStats) getTotalOutputBytes() uint64 { return atomic.LoadUint64(&s.totalOutputBytes) } -// Increase outbound input bytes +// Increase S3 total input bytes func (s *ConnStats) incS3InputBytes(n int64) { atomic.AddUint64(&s.s3InputBytes, uint64(n)) } -// Increase outbound output bytes +// Increase S3 total output bytes func (s *ConnStats) incS3OutputBytes(n int64) { atomic.AddUint64(&s.s3OutputBytes, uint64(n)) } -// Return outbound input bytes +// Return S3 total input bytes func (s *ConnStats) getS3InputBytes() uint64 { return atomic.LoadUint64(&s.s3InputBytes) } -// Return outbound output bytes +// Return S3 total output bytes func (s *ConnStats) getS3OutputBytes() uint64 { return atomic.LoadUint64(&s.s3OutputBytes) } +// Increase Admin total input bytes +func (s *ConnStats) incAdminInputBytes(n int64) { + atomic.AddUint64(&s.adminInputBytes, uint64(n)) +} + +// Increase Admin total output bytes +func (s *ConnStats) incAdminOutputBytes(n int64) { + atomic.AddUint64(&s.adminOutputBytes, uint64(n)) +} + +// Return Admin total input bytes +func (s *ConnStats) getAdminInputBytes() uint64 { + return atomic.LoadUint64(&s.adminInputBytes) +} + +// Return Admin total output bytes +func (s *ConnStats) getAdminOutputBytes() uint64 { + return atomic.LoadUint64(&s.adminOutputBytes) +} + // Return connection stats (total input/output bytes and total s3 input/output bytes) func (s *ConnStats) toServerConnStats() ServerConnStats { return ServerConnStats{ - TotalInputBytes: s.getTotalInputBytes(), // Traffic including reserved bucket - TotalOutputBytes: s.getTotalOutputBytes(), // Traffic including reserved bucket - S3InputBytes: s.getS3InputBytes(), // Traffic for client buckets - S3OutputBytes: s.getS3OutputBytes(), // Traffic for client buckets + TotalInputBytes: s.getTotalInputBytes(), // Traffic internode received + TotalOutputBytes: s.getTotalOutputBytes(), // Traffic internode sent + S3InputBytes: s.getS3InputBytes(), // Traffic S3 received + S3OutputBytes: s.getS3OutputBytes(), // Traffic S3 sent + AdminInputBytes: s.getAdminInputBytes(), // Traffic admin calls received + AdminOutputBytes: s.getAdminOutputBytes(), // Traffic admin calls sent } } @@ -92,6 +116,84 @@ func newConnStats() *ConnStats { return &ConnStats{} } +type bucketS3RXTX struct { + s3InputBytes uint64 + s3OutputBytes uint64 +} + +type bucketConnStats struct { + sync.RWMutex + stats map[string]*bucketS3RXTX +} + +func newBucketConnStats() *bucketConnStats { + return &bucketConnStats{ + stats: make(map[string]*bucketS3RXTX), + } +} + +// Increase S3 total input bytes for input bucket +func (s *bucketConnStats) incS3InputBytes(bucket string, n int64) { + s.Lock() + defer s.Unlock() + stats, ok := s.stats[bucket] + if !ok { + stats = &bucketS3RXTX{ + s3InputBytes: uint64(n), + } + } else { + stats.s3InputBytes += uint64(n) + } + s.stats[bucket] = stats +} + +// Increase S3 total output bytes for input bucket +func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) { + s.Lock() + defer s.Unlock() + stats, ok := s.stats[bucket] + if !ok { + stats = &bucketS3RXTX{ + s3OutputBytes: uint64(n), + } + } else { + stats.s3OutputBytes += uint64(n) + } + s.stats[bucket] = stats +} + +// Return S3 total input bytes for input bucket +func (s *bucketConnStats) getS3InputBytes(bucket string) uint64 { + s.RLock() + defer s.RUnlock() + + stats := s.stats[bucket] + if stats == nil { + return 0 + } + return stats.s3InputBytes +} + +// Return S3 total output bytes +func (s *bucketConnStats) getS3OutputBytes(bucket string) uint64 { + s.RLock() + defer s.RUnlock() + + stats := s.stats[bucket] + if stats == nil { + return 0 + } + return stats.s3OutputBytes +} + +// delete metrics once bucket is deleted. +func (s *bucketConnStats) delete(bucket string) { + s.Lock() + defer s.Unlock() + + delete(s.stats, bucket) +} + // HTTPAPIStats holds statistics information about // a given API in the requests. type HTTPAPIStats struct { diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index cc43a2000..d7dec7957 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -293,7 +293,7 @@ func getClusterCapacityTotalBytesMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: capacityRawSubsystem, Name: totalBytes, - Help: "Total capacity online in the cluster.", + Help: "Total capacity online in the cluster", Type: gaugeMetric, } } @@ -303,7 +303,7 @@ func getClusterCapacityFreeBytesMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: capacityRawSubsystem, Name: freeBytes, - Help: "Total free capacity online in the cluster.", + Help: "Total free capacity online in the cluster", Type: gaugeMetric, } } @@ -313,7 +313,7 @@ func getClusterCapacityUsageBytesMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: capacityUsableSubsystem, Name: totalBytes, - Help: "Total usable capacity online in the cluster.", + Help: "Total usable capacity online in the cluster", Type: gaugeMetric, } } @@ -323,7 +323,7 @@ func getClusterCapacityUsageFreeBytesMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: capacityUsableSubsystem, Name: freeBytes, - Help: "Total free usable capacity online in the cluster.", + Help: "Total free usable capacity online in the cluster", Type: gaugeMetric, } } @@ -333,7 +333,7 @@ func getNodeDiskAPILatencyMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: diskSubsystem, Name: apiLatencyMicroSec, - Help: "Average last minute latency in µs for disk API storage operations.", + Help: "Average last minute latency in µs for disk API storage operations", Type: gaugeMetric, } } @@ -343,7 +343,7 @@ func getNodeDiskUsedBytesMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: diskSubsystem, Name: usedBytes, - Help: "Total storage used on a disk.", + Help: "Total storage used on a disk", Type: gaugeMetric, } } @@ -353,7 +353,7 @@ func getNodeDiskFreeBytesMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: diskSubsystem, Name: freeBytes, - Help: "Total storage available on a disk.", + Help: "Total storage available on a disk", Type: gaugeMetric, } } @@ -363,7 +363,7 @@ func getClusterDisksOfflineTotalMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: diskSubsystem, Name: offlineTotal, - Help: "Total disks offline.", + Help: "Total disks offline", Type: gaugeMetric, } } @@ -373,7 +373,7 @@ func getClusterDisksOnlineTotalMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: diskSubsystem, Name: onlineTotal, - Help: "Total disks online.", + Help: "Total disks online", Type: gaugeMetric, } } @@ -383,7 +383,7 @@ func getClusterDisksTotalMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: diskSubsystem, Name: total, - Help: "Total disks.", + Help: "Total disks", Type: gaugeMetric, } } @@ -393,7 +393,7 @@ func getClusterDisksFreeInodes() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: diskSubsystem, Name: freeInodes, - Help: "Total free inodes.", + Help: "Total free inodes", Type: gaugeMetric, } } @@ -403,7 +403,7 @@ func getNodeDiskTotalBytesMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: diskSubsystem, Name: totalBytes, - Help: "Total storage on a disk.", + Help: "Total storage on a disk", Type: gaugeMetric, } } @@ -428,6 +428,26 @@ func getBucketUsageQuotaTotalBytesMD() MetricDescription { } } +func getBucketTrafficReceivedBytes() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: trafficSubsystem, + Name: receivedBytes, + Help: "Total number of S3 bytes received for this bucket", + Type: gaugeMetric, + } +} + +func getBucketTrafficSentBytes() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: trafficSubsystem, + Name: sentBytes, + Help: "Total number of S3 bytes sent for this bucket", + Type: gaugeMetric, + } +} + func getBucketUsageTotalBytesMD() MetricDescription { return MetricDescription{ Namespace: bucketMetricNamespace, @@ -453,7 +473,7 @@ func getBucketRepLatencyMD() MetricDescription { Namespace: bucketMetricNamespace, Subsystem: replicationSubsystem, Name: latencyMilliSec, - Help: "Replication latency in milliseconds.", + Help: "Replication latency in milliseconds", Type: histogramMetric, } } @@ -463,7 +483,7 @@ func getBucketRepFailedBytesMD() MetricDescription { Namespace: bucketMetricNamespace, Subsystem: replicationSubsystem, Name: failedBytes, - Help: "Total number of bytes failed at least once to replicate.", + Help: "Total number of bytes failed at least once to replicate", Type: gaugeMetric, } } @@ -473,7 +493,7 @@ func getBucketRepSentBytesMD() MetricDescription { Namespace: bucketMetricNamespace, Subsystem: replicationSubsystem, Name: sentBytes, - Help: "Total number of bytes replicated to the target bucket.", + Help: "Total number of bytes replicated to the target bucket", Type: gaugeMetric, } } @@ -483,7 +503,7 @@ func getBucketRepReceivedBytesMD() MetricDescription { Namespace: bucketMetricNamespace, Subsystem: replicationSubsystem, Name: receivedBytes, - Help: "Total number of bytes replicated to this bucket from another source bucket.", + Help: "Total number of bytes replicated to this bucket from another source bucket", Type: gaugeMetric, } } @@ -503,7 +523,7 @@ func getBucketObjectDistributionMD() MetricDescription { Namespace: bucketMetricNamespace, Subsystem: objectsSubsystem, Name: sizeDistribution, - Help: "Distribution of object sizes in the bucket, includes label for the bucket name.", + Help: "Distribution of object sizes in the bucket, includes label for the bucket name", Type: histogramMetric, } } @@ -513,7 +533,7 @@ func getInternodeFailedRequests() MetricDescription { Namespace: interNodeMetricNamespace, Subsystem: trafficSubsystem, Name: errorsTotal, - Help: "Total number of failed internode calls.", + Help: "Total number of failed internode calls", Type: counterMetric, } } @@ -523,7 +543,7 @@ func getInterNodeSentBytesMD() MetricDescription { Namespace: interNodeMetricNamespace, Subsystem: trafficSubsystem, Name: sentBytes, - Help: "Total number of bytes sent to the other peer nodes.", + Help: "Total number of bytes sent to the other peer nodes", Type: counterMetric, } } @@ -533,7 +553,7 @@ func getInterNodeReceivedBytesMD() MetricDescription { Namespace: interNodeMetricNamespace, Subsystem: trafficSubsystem, Name: receivedBytes, - Help: "Total number of bytes received from other peer nodes.", + Help: "Total number of bytes received from other peer nodes", Type: counterMetric, } } @@ -553,7 +573,7 @@ func getS3ReceivedBytesMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: trafficSubsystem, Name: receivedBytes, - Help: "Total number of s3 bytes received.", + Help: "Total number of s3 bytes received", Type: counterMetric, } } @@ -643,7 +663,7 @@ func getS3RejectedAuthRequestsTotalMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: requestsRejectedSubsystem, Name: authTotal, - Help: "Total number S3 requests rejected for auth failure.", + Help: "Total number S3 requests rejected for auth failure", Type: counterMetric, } } @@ -653,7 +673,7 @@ func getS3RejectedHeaderRequestsTotalMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: requestsRejectedSubsystem, Name: headerTotal, - Help: "Total number S3 requests rejected for invalid header.", + Help: "Total number S3 requests rejected for invalid header", Type: counterMetric, } } @@ -663,7 +683,7 @@ func getS3RejectedTimestampRequestsTotalMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: requestsRejectedSubsystem, Name: timestampTotal, - Help: "Total number S3 requests rejected for invalid timestamp.", + Help: "Total number S3 requests rejected for invalid timestamp", Type: counterMetric, } } @@ -673,7 +693,7 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: requestsRejectedSubsystem, Name: invalidTotal, - Help: "Total number S3 invalid requests.", + Help: "Total number S3 invalid requests", Type: counterMetric, } } @@ -793,7 +813,7 @@ func getNodeOnlineTotalMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: nodesSubsystem, Name: onlineTotal, - Help: "Total number of MinIO nodes online.", + Help: "Total number of MinIO nodes online", Type: gaugeMetric, } } @@ -803,7 +823,7 @@ func getNodeOfflineTotalMD() MetricDescription { Namespace: clusterMetricNamespace, Subsystem: nodesSubsystem, Name: offlineTotal, - Help: "Total number of MinIO nodes offline.", + Help: "Total number of MinIO nodes offline", Type: gaugeMetric, } } @@ -823,7 +843,7 @@ func getMinIOCommitMD() MetricDescription { Namespace: minioMetricNamespace, Subsystem: softwareSubsystem, Name: commitInfo, - Help: "Git commit hash for the MinIO release.", + Help: "Git commit hash for the MinIO release", Type: gaugeMetric, } } @@ -833,7 +853,7 @@ func getS3TTFBDistributionMD() MetricDescription { Namespace: s3MetricNamespace, Subsystem: timeSubsystem, Name: ttfbDistribution, - Help: "Distribution of the time to first byte across API calls.", + Help: "Distribution of the time to first byte across API calls", Type: gaugeMetric, } } @@ -843,7 +863,7 @@ func getMinioFDOpenMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: fileDescriptorSubsystem, Name: openTotal, - Help: "Total number of open file descriptors by the MinIO Server process.", + Help: "Total number of open file descriptors by the MinIO Server process", Type: gaugeMetric, } } @@ -853,7 +873,7 @@ func getMinioFDLimitMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: fileDescriptorSubsystem, Name: limitTotal, - Help: "Limit on total number of open file descriptors for the MinIO Server process.", + Help: "Limit on total number of open file descriptors for the MinIO Server process", Type: gaugeMetric, } } @@ -923,7 +943,7 @@ func getMinIOGORoutineCountMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: goRoutines, Name: total, - Help: "Total number of go routines running.", + Help: "Total number of go routines running", Type: gaugeMetric, } } @@ -933,7 +953,7 @@ func getMinIOProcessStartTimeMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: processSubsystem, Name: startTime, - Help: "Start time for MinIO process per node, time in seconds since Unix epoc.", + Help: "Start time for MinIO process per node, time in seconds since Unix epoc", Type: gaugeMetric, } } @@ -943,7 +963,7 @@ func getMinIOProcessUptimeMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: processSubsystem, Name: upTime, - Help: "Uptime for MinIO process per node in seconds.", + Help: "Uptime for MinIO process per node in seconds", Type: gaugeMetric, } } @@ -953,7 +973,7 @@ func getMinIOProcessResidentMemory() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: processSubsystem, Name: memory, - Help: "Resident memory size in bytes.", + Help: "Resident memory size in bytes", Type: gaugeMetric, } } @@ -963,7 +983,7 @@ func getMinIOProcessCPUTime() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: processSubsystem, Name: cpu, - Help: "Total user and system CPU time spent in seconds.", + Help: "Total user and system CPU time spent in seconds", Type: counterMetric, } } @@ -1148,7 +1168,7 @@ func getTransitionPendingTasksMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: ilmSubsystem, Name: transitionPendingTasks, - Help: "Number of pending ILM transition tasks in the queue.", + Help: "Number of pending ILM transition tasks in the queue", Type: gaugeMetric, } } @@ -1158,7 +1178,7 @@ func getTransitionActiveTasksMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: ilmSubsystem, Name: transitionActiveTasks, - Help: "Number of active ILM transition tasks.", + Help: "Number of active ILM transition tasks", Type: gaugeMetric, } } @@ -1168,7 +1188,7 @@ func getExpiryPendingTasksMD() MetricDescription { Namespace: nodeMetricNamespace, Subsystem: ilmSubsystem, Name: expiryPendingTasks, - Help: "Number of pending ILM expiry tasks in the queue.", + Help: "Number of pending ILM expiry tasks in the queue", Type: gaugeMetric, } } @@ -1210,7 +1230,7 @@ func getScannerNodeMetrics() *MetricsGroup { Namespace: nodeMetricNamespace, Subsystem: scannerSubsystem, Name: "objects_scanned", - Help: "Total number of unique objects scanned since server start.", + Help: "Total number of unique objects scanned since server start", Type: counterMetric, }, Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)), @@ -1220,7 +1240,7 @@ func getScannerNodeMetrics() *MetricsGroup { Namespace: nodeMetricNamespace, Subsystem: scannerSubsystem, Name: "versions_scanned", - Help: "Total number of object versions scanned since server start.", + Help: "Total number of object versions scanned since server start", Type: counterMetric, }, Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)), @@ -1230,7 +1250,7 @@ func getScannerNodeMetrics() *MetricsGroup { Namespace: nodeMetricNamespace, Subsystem: scannerSubsystem, Name: "directories_scanned", - Help: "Total number of directories scanned since server start.", + Help: "Total number of directories scanned since server start", Type: counterMetric, }, Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)), @@ -1674,6 +1694,24 @@ func getBucketUsageMetrics() *MetricsGroup { }) } + recvBytes := globalBucketConnStats.getS3InputBytes(bucket) + if recvBytes > 0 { + metrics = append(metrics, Metric{ + Description: getBucketTrafficReceivedBytes(), + Value: float64(recvBytes), + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + + sentBytes := globalBucketConnStats.getS3OutputBytes(bucket) + if sentBytes > 0 { + metrics = append(metrics, Metric{ + Description: getBucketTrafficSentBytes(), + Value: float64(sentBytes), + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + if stats.hasReplicationUsage() { for arn, stat := range stats.Stats { metrics = append(metrics, Metric{ diff --git a/cmd/notification.go b/cmd/notification.go index c697da8cb..9dfa8ea4d 100644 --- a/cmd/notification.go +++ b/cmd/notification.go @@ -579,6 +579,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName globalBucketMetadataSys.Remove(bucketName) globalBucketTargetSys.Delete(bucketName) globalNotificationSys.RemoveNotification(bucketName) + globalBucketConnStats.delete(bucketName) if localMetacacheMgr != nil { localMetacacheMgr.deleteBucketCache(bucketName) } diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go index 0a551f201..72541de2d 100644 --- a/cmd/peer-rest-server.go +++ b/cmd/peer-rest-server.go @@ -538,6 +538,8 @@ func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *h globalReplicationStats.Delete(bucketName) globalBucketMetadataSys.Remove(bucketName) globalBucketTargetSys.Delete(bucketName) + globalNotificationSys.RemoveNotification(bucketName) + globalBucketConnStats.delete(bucketName) if localMetacacheMgr != nil { localMetacacheMgr.deleteBucketCache(bucketName) } diff --git a/docs/metrics/prometheus/list.md b/docs/metrics/prometheus/list.md index 98fe647a0..ec2b59d01 100644 --- a/docs/metrics/prometheus/list.md +++ b/docs/metrics/prometheus/list.md @@ -15,6 +15,8 @@ These metrics can be from any MinIO server once per collection. | `minio_bucket_usage_object_total` | Total number of objects | | `minio_bucket_usage_total_bytes` | Total bucket size in bytes | | `minio_bucket_quota_total_bytes` | Total bucket quota size in bytes | +| `minio_bucket_traffic_sent_bytes` | Total s3 bytes sent per bucket | +| `minio_bucket_traffic_received_bytes` | Total s3 bytes received per bucket | | `minio_cache_hits_total` | Total number of disk cache hits | | `minio_cache_missed_total` | Total number of disk cache misses | | `minio_cache_sent_bytes` | Total number of bytes served from cache |