self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-10-02 22:24:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

metric: add KMS-related metrics (#15258)

Browse source 
This commit adds a minimal set of KMS-related metrics:
```
 # HELP minio_cluster_kms_online Reports whether the KMS is online (1) or offline (0)
 # TYPE minio_cluster_kms_online gauge
 minio_cluster_kms_online{server="127.0.0.1:9000"} 1
 # HELP minio_cluster_kms_request_error Number of KMS requests that failed with a well-defined error
 # TYPE minio_cluster_kms_request_error counter
 minio_cluster_kms_request_error{server="127.0.0.1:9000"} 16790
 # HELP minio_cluster_kms_request_success Number of KMS requests that succeeded
 # TYPE minio_cluster_kms_request_success counter
 minio_cluster_kms_request_success{server="127.0.0.1:9000"} 348031
```

Currently, we report whether the KMS is available and how many requests
succeeded/failed. However, KES exposes much more metrics that can be
exposed if necessary. See: https://pkg.go.dev/github.com/minio/kes#Metric

Signed-off-by: Andreas Auernhammer <hi@aead.dev>
This commit is contained in:
Andreas Auernhammer 2022-07-11 18:17:28 +02:00 committed by GitHub
parent b49fc33cb3
commit f800cee4fa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 130 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
cmd/metrics-v2.go
View file

@ -27,6 +27,7 @@ import (
"sync/atomic"
"time"
"github.com/minio/kes"
"github.com/minio/madmin-go"
"github.com/minio/minio/internal/bucket/lifecycle"
"github.com/minio/minio/internal/logger"
@ -49,6 +50,7 @@ func init() {
getNodeHealthMetrics(),
getClusterStorageMetrics(),
getClusterTierMetrics(),
getKMSMetrics(),
}
peerMetricsGroups = []*MetricsGroup{
@ -63,6 +65,7 @@ func init() {
getILMNodeMetrics(),
getScannerNodeMetrics(),
getIAMNodeMetrics(),
getKMSNodeMetrics(),
}
allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@ -80,7 +83,6 @@ func init() {
getMinioVersionMetrics(),
getS3TTFBMetric(),
})
clusterCollector = newMinioClusterCollector(allMetricsGroups)
}
@ -123,6 +125,7 @@ const (
ilmSubsystem MetricSubsystem = "ilm"
scannerSubsystem MetricSubsystem = "scanner"
iamSubsystem MetricSubsystem = "iam"
kmsSubsystem MetricSubsystem = "kms"
)
// MetricName are the individual names for the metric.
@ -188,6 +191,12 @@ const (
transitionedBytes MetricName = "transitioned_bytes"
transitionedObjects MetricName = "transitioned_objects"
transitionedVersions MetricName = "transitioned_versions"
kmsOnline = "online"
kmsRequestsSuccess = "request_success"
kmsRequestsError = "request_error"
kmsRequestsFail = "request_failure"
kmsUptime = "uptime"
)
const (
@ -1931,6 +1940,107 @@ func getClusterStorageMetrics() *MetricsGroup {
return mg
}
func getKMSNodeMetrics() *MetricsGroup {
mg := &MetricsGroup{
cacheInterval: 10 * time.Second,
}
mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
objLayer := newObjectLayerFn()
// Service not initialized yet
if objLayer == nil || globalIsGateway || GlobalKMS == nil {
return
}
const (
Online = 1
Offline = 0
)
desc := MetricDescription{
Namespace: clusterMetricNamespace,
Subsystem: kmsSubsystem,
Name: kmsOnline,
Help: "Reports whether the KMS is online (1) or offline (0)",
Type: gaugeMetric,
}
_, err := GlobalKMS.Metrics(ctx)
if _, ok := kes.IsConnError(err); ok {
return []Metric{{
Description: desc,
Value: float64(Offline),
}}
}
return []Metric{{
Description: desc,
Value: float64(Online),
}}
})
return mg
}
func getKMSMetrics() *MetricsGroup {
mg := &MetricsGroup{
cacheInterval: 10 * time.Second,
}
mg.RegisterRead(func(ctx context.Context) []Metric {
objLayer := newObjectLayerFn()
// Service not initialized yet
if objLayer == nil || globalIsGateway || GlobalKMS == nil {
return []Metric{}
}
metrics := make([]Metric, 0, 4)
metric, err := GlobalKMS.Metrics(ctx)
if err != nil {
return metrics
}
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: clusterMetricNamespace,
Subsystem: kmsSubsystem,
Name: kmsRequestsSuccess,
Help: "Number of KMS requests that succeeded",
Type: counterMetric,
},
Value: float64(metric.RequestOK),
})
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: clusterMetricNamespace,
Subsystem: kmsSubsystem,
Name: kmsRequestsError,
Help: "Number of KMS requests that failed due to some error. (HTTP 4xx status code)",
Type: counterMetric,
},
Value: float64(metric.RequestErr),
})
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: clusterMetricNamespace,
Subsystem: kmsSubsystem,
Name: kmsRequestsFail,
Help: "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)",
Type: counterMetric,
},
Value: float64(metric.RequestFail),
})
metrics = append(metrics, Metric{
Description: MetricDescription{
Namespace: clusterMetricNamespace,
Subsystem: kmsSubsystem,
Name: kmsUptime,
Help: "The time the KMS has been up and running in seconds.",
Type: counterMetric,
},
Value: metric.UpTime.Seconds(),
})
return metrics
})
return mg
}
type minioClusterCollector struct {
metricsGroups []*MetricsGroup
desc *prometheus.Desc

4
go.mod
View file

@ -47,7 +47,7 @@ require (
github.com/minio/csvparser v1.0.0
github.com/minio/dperf v0.4.2
github.com/minio/highwayhash v1.0.2
github.com/minio/kes v0.19.2
github.com/minio/kes v0.20.0
github.com/minio/madmin-go v1.4.3
github.com/minio/minio-go/v7 v7.0.30
github.com/minio/pkg v1.1.26
@ -65,7 +65,7 @@ require (
github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
github.com/pierrec/lz4 v2.6.1+incompatible
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_golang v1.12.2
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.34.0
github.com/prometheus/procfs v0.7.3

7
go.sum
View file

@ -618,8 +618,8 @@ github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEX
github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/kes v0.19.2 h1:0kdMAgLMSkiDA33k8pMHC7d6erDuseuLrZF+N3017SM=
github.com/minio/kes v0.19.2/go.mod h1:X2fMkDbAkjbSKDGOQZvyPkHxoG7nuzP6R78Jw+TzXtM=
github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
github.com/minio/madmin-go v1.4.3 h1:5/kBHjKTjYOQQHvyznu51weN5hJtFW67LB2VLz+hmzU=
github.com/minio/madmin-go v1.4.3/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
@ -745,8 +745,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

4
internal/kms/kes.go
View file

@ -115,6 +115,10 @@ func (c *kesClient) Stat() (Status, error) {
}, nil
}
func (c *kesClient) Metrics(ctx context.Context) (kes.Metric, error) {
return c.client.Metrics(ctx)
}
// CreateKey tries to create a new key at the KMS with the
// given key ID.
//

4
internal/kms/kms.go
View file

@ -23,6 +23,7 @@ import (
"encoding/json"
jsoniter "github.com/json-iterator/go"
"github.com/minio/kes"
)
// KMS is the generic interface that abstracts over
@ -31,6 +32,9 @@ type KMS interface {
// Stat returns the current KMS status.
Stat() (Status, error)
// Metrics returns a KMS metric snapshot.
Metrics(ctx context.Context) (kes.Metric, error)
// CreateKey creates a new key at the KMS with the given key ID.
CreateKey(keyID string) error

5
internal/kms/single-key.go
View file

@ -33,6 +33,7 @@ import (
"golang.org/x/crypto/chacha20"
"golang.org/x/crypto/chacha20poly1305"
"github.com/minio/kes"
"github.com/minio/minio/internal/hash/sha256"
)
@ -89,6 +90,10 @@ func (kms secretKey) Stat() (Status, error) {
}, nil
}
func (secretKey) Metrics(ctx context.Context) (kes.Metric, error) {
return kes.Metric{}, errors.New("kms: metrics are not supported")
}
func (secretKey) CreateKey(string) error {
return errors.New("kms: creating keys is not supported")
}