Include speedtest as part of healthinfo api (#14696)

Execute the object, drive and net speedtests as part of the healthinfo
(if requested by the client), and include their result in the response.

The options for the speedtests have been picked from the default values
used by `mc support perf` command.
This commit is contained in:
Shireesh Anjal 2022-04-13 01:47:44 +05:30 committed by GitHub
parent 5f94cec1e2
commit 5c53620a72
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 152 additions and 80 deletions

View file

@ -1084,8 +1084,6 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
return
}
var bucketExists bool
sizeStr := r.Form.Get(peerRESTSize)
durationStr := r.Form.Get(peerRESTDuration)
concurrentStr := r.Form.Get(peerRESTConcurrent)
@ -1111,49 +1109,29 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
duration = time.Second * 10
}
// ignores any errors here.
storageInfo, _ := objectAPI.StorageInfo(ctx)
capacityNeeded := uint64(concurrent * size)
capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
if capacity < capacityNeeded {
if !sufficientCapacity {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{
Code: "XMinioSpeedtestInsufficientCapacity",
Message: fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)),
Code: "XMinioSpeedtestInsufficientCapacity",
Message: capacityErrMsg,
StatusCode: http.StatusInsufficientStorage,
}), r.URL)
return
}
// Verify if we can employ autotune without running out of capacity,
// if we do run out of capacity, make sure to turn-off autotuning
// in such situations.
newConcurrent := concurrent + (concurrent+1)/2
autoTunedCapacityNeeded := uint64(newConcurrent * size)
if autotune && capacity < autoTunedCapacityNeeded {
// Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity.
if autotune && !canAutotune {
autotune = false
}
err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
if err != nil {
if _, ok := err.(BucketExists); !ok {
// Only BucketExists error can be ignored.
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
bucketExists = true
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
return
}
deleteBucket := func() {
objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
Force: true,
NoRecreate: true,
})
}
if !bucketExists {
defer deleteBucket()
defer deleteObjectPerfBucket(objectAPI)
}
// Freeze all incoming S3 API calls before running speedtest.
@ -1166,7 +1144,7 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
defer keepAliveTicker.Stop()
enc := json.NewEncoder(w)
ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
ch := objectSpeedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
for {
select {
case <-ctx.Done():
@ -1189,6 +1167,50 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
}
}
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer) (bucketExists bool, err error) {
err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
if err != nil {
if _, ok := err.(BucketExists); !ok {
// Only BucketExists error can be ignored.
return false, err
}
bucketExists = true
}
return bucketExists, nil
}
func deleteObjectPerfBucket(objectAPI ObjectLayer) {
objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
Force: true,
NoRecreate: true,
})
}
func validateObjPerfOptions(ctx context.Context, objectAPI ObjectLayer, concurrent int, size int, autotune bool) (sufficientCapacity bool, canAutotune bool, capacityErrMsg string) {
storageInfo, _ := objectAPI.StorageInfo(ctx)
capacityNeeded := uint64(concurrent * size)
capacity := uint64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))
if capacity < capacityNeeded {
return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s",
humanize.IBytes(capacityNeeded), humanize.IBytes(capacity))
}
// Verify if we can employ autotune without running out of capacity,
// if we do run out of capacity, make sure to turn-off autotuning
// in such situations.
if autotune {
newConcurrent := concurrent + (concurrent+1)/2
autoTunedCapacityNeeded := uint64(newConcurrent * size)
if capacity < autoTunedCapacityNeeded {
// Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity.
return true, false, ""
}
}
return true, autotune, ""
}
// NetSpeedtestHandler - reports maximum network throughput
func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
ctx := newContext(r, w, "NetSpeedtestHandler")
@ -1787,7 +1809,9 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
}
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
if query.Get("perfnet") != "" || query.Get("perfdrive") != "" {
if query.Get(string(madmin.HealthDataTypePerfNet)) != "" ||
query.Get(string(madmin.HealthDataTypePerfDrive)) != "" ||
query.Get(string(madmin.HealthDataTypePerfObj)) != "" {
deadline = 1 * time.Hour
}
if dstr := r.Form.Get("deadline"); dstr != "" {
@ -2032,49 +2056,98 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
}
getAndWriteDrivePerfInfo := func() {
if query.Get("perfdrive") == "true" {
localDPI := getDrivePerfInfos(deadlinedCtx, globalLocalNodeName)
anonymizeAddr(&localDPI)
healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, localDPI)
partialWrite(healthInfo)
if query.Get(string(madmin.HealthDataTypePerfDrive)) == "true" {
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// unfreeze all incoming S3 API calls after speedtest.
defer globalNotificationSys.ServiceFreeze(ctx, false)
perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx)
for perfInfo := range perfCh {
anonymizeAddr(&perfInfo)
healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo)
partialWrite(healthInfo)
opts := madmin.DriveSpeedTestOpts{
Serial: false,
BlockSize: 4 * humanize.MiByte,
FileSize: 1 * humanize.GiByte,
}
localDPI := driveSpeedTest(ctx, opts)
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, localDPI)
perfCh := globalNotificationSys.DriveSpeedTest(ctx, opts)
for perfInfo := range perfCh {
healthInfo.Perf.DrivePerf = append(healthInfo.Perf.DrivePerf, perfInfo)
}
partialWrite(healthInfo)
}
}
anonymizeNetPerfInfo := func(npi *madmin.NetPerfInfo) {
anonymizeAddr(npi)
rps := npi.RemotePeers
for idx, peer := range rps {
anonymizeAddr(&peer)
rps[idx] = peer
getAndWriteObjPerfInfo := func() {
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
concurrent := 32
size := 64 * humanize.MiByte
autotune := true
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, objectAPI, concurrent, size, autotune)
if !sufficientCapacity {
healthInfo.Perf.Error = capacityErrMsg
partialWrite(healthInfo)
return
}
if !canAutotune {
autotune = false
}
bucketExists, err := makeObjectPerfBucket(ctx, objectAPI)
if err != nil {
healthInfo.Perf.Error = "Could not make object perf bucket: " + err.Error()
partialWrite(healthInfo)
return
}
if !bucketExists {
defer deleteObjectPerfBucket(objectAPI)
}
// Freeze all incoming S3 API calls before running speedtest.
globalNotificationSys.ServiceFreeze(ctx, true)
// unfreeze all incoming S3 API calls after speedtest.
defer globalNotificationSys.ServiceFreeze(ctx, false)
opts := speedTestOpts{
throughputSize: size,
concurrencyStart: concurrent,
duration: 10 * time.Second,
autotune: autotune,
}
perfCh := objectSpeedTest(ctx, opts)
for perfInfo := range perfCh {
healthInfo.Perf.ObjPerf = append(healthInfo.Perf.ObjPerf, perfInfo)
}
partialWrite(healthInfo)
}
npi.RemotePeers = rps
}
getAndWriteNetPerfInfo := func() {
if globalIsDistErasure && query.Get("perfnet") == "true" {
localNPI := globalNotificationSys.GetNetPerfInfo(deadlinedCtx)
anonymizeNetPerfInfo(&localNPI)
healthInfo.Perf.Net = append(healthInfo.Perf.Net, localNPI)
partialWrite(healthInfo)
netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
for netInfo := range netInfos {
anonymizeNetPerfInfo(&netInfo)
healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
partialWrite(healthInfo)
if query.Get(string(madmin.HealthDataTypePerfObj)) == "true" {
if !globalIsDistErasure {
return
}
ppi := globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx)
anonymizeNetPerfInfo(&ppi)
healthInfo.Perf.NetParallel = ppi
nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
if err != nil {
healthInfo.Perf.Error = "Could not acquire lock for netperf: " + err.Error()
} else {
defer nsLock.Unlock(lkctx.Cancel)
netPerf := globalNotificationSys.Netperf(ctx, time.Second*10)
for _, np := range netPerf {
np.Endpoint = anonAddr(np.Endpoint)
healthInfo.Perf.NetPerf = append(healthInfo.Perf.NetPerf, np)
}
}
partialWrite(healthInfo)
}
}
@ -2108,6 +2181,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
getAndWriteProcInfo()
getAndWriteMinioConfig()
getAndWriteDrivePerfInfo()
getAndWriteObjPerfInfo()
getAndWriteNetPerfInfo()
getAndWriteSysErrors()
getAndWriteSysServices()
@ -2640,6 +2714,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
if !found {
// In distributed setup, anonymized addr = 'poolNum.serverNum'
newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum)
schemePfx := endpoint.Scheme + "://"
// Hostname
mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost)
@ -2649,6 +2724,7 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
// Host + port
newHostPort = newHost + ":" + endpoint.Port()
mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort)
mapIfNotPresent(hostAnonymizer, schemePfx+endpoint.Host, newHostPort)
}
newHostPortPath := newHostPort
@ -2657,10 +2733,11 @@ func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum
currentHostPortPath := endpoint.Host + endpoint.Path
newHostPortPath = newHostPort + endpoint.Path
mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath)
mapIfNotPresent(hostAnonymizer, schemePfx+currentHostPortPath, newHostPortPath)
}
// Full url
hostAnonymizer[currentURL] = endpoint.Scheme + "://" + newHostPortPath
hostAnonymizer[currentURL] = schemePfx + newHostPortPath
}
}

View file

@ -156,11 +156,6 @@ func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedT
return ch
}
// speedTest - Deprecated. See objectSpeedTest
func speedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedTestResult {
return objectSpeedTest(ctx, opts)
}
func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.DriveSpeedTestResult {
perf := &dperf.DrivePerf{
Serial: opts.Serial,

6
go.mod
View file

@ -50,7 +50,7 @@ require (
github.com/minio/dperf v0.3.5
github.com/minio/highwayhash v1.0.2
github.com/minio/kes v0.19.0
github.com/minio/madmin-go v1.3.6
github.com/minio/madmin-go v1.3.11
github.com/minio/minio-go/v7 v7.0.23
github.com/minio/parquet-go v1.1.0
github.com/minio/pkg v1.1.20
@ -105,6 +105,7 @@ require (
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
github.com/briandowns/spinner v1.18.1 // indirect
github.com/charmbracelet/bubbles v0.10.3 // indirect
github.com/charmbracelet/bubbletea v0.20.0 // indirect
github.com/charmbracelet/lipgloss v0.5.0 // indirect
@ -167,10 +168,9 @@ require (
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/mattn/go-runewidth v0.0.13 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/minio/argon2 v1.0.0 // indirect
github.com/minio/colorjson v1.0.1 // indirect
github.com/minio/filepath v1.0.0 // indirect
github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect
github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.4.3 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

12
go.sum
View file

@ -139,6 +139,8 @@ github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
github.com/bits-and-blooms/bloom/v3 v3.0.1 h1:Inlf0YXbgehxVjMPmCGv86iMCKMGPPrPSHtBF5yRHwA=
github.com/bits-and-blooms/bloom/v3 v3.0.1/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/briandowns/spinner v1.18.1 h1:yhQmQtM1zsqFsouh09Bk/jCjd50pC3EOGsh28gLVvwY=
github.com/briandowns/spinner v1.18.1/go.mod h1:mQak9GHqbspjC/5iUx3qMlIho8xBS/ppAL/hX5SmPJU=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
@ -657,8 +659,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.1.46 h1:uzwpxRtSVxtcIZmz/4Uz6/Rn7G11DvsaslXoy5LxQio=
github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
github.com/minio/argon2 v1.0.0 h1:cLB/fl0EeBqiDYhsIzIPTdLZhCykRrvdx3Eu3E5oqsE=
github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ=
github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
@ -676,10 +676,10 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/kes v0.19.0 h1:rKzkDXT4ay7FBW34KgXK+y85bie4x4Oiq29ONRuMzh0=
github.com/minio/kes v0.19.0/go.mod h1:e9YGKbwFCV7LbqNPMfZBazfNUsFGJ5LG4plSeWL8mmg=
github.com/minio/madmin-go v1.3.6 h1:lgAIfOI1xB+K2JFx5JqxklLS0SE7ya3qZIi1qmwjHNE=
github.com/minio/madmin-go v1.3.6/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
github.com/minio/mc v0.0.0-20220302011226-f13defa54577 h1:ZWkSq/Ofoz7fX4syAscCg66Ie6oV5plYFTXAieYRRDg=
github.com/minio/mc v0.0.0-20220302011226-f13defa54577/go.mod h1:lomZcm7bdn9Og7UBKBqu//8YPmWAhhMNlnkIAHmwcS8=
github.com/minio/madmin-go v1.3.11 h1:Cj02kzG2SD1pnZW2n1joe00yqb6NFE40Jt2gp+5mWFQ=
github.com/minio/madmin-go v1.3.11/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054 h1:Od5VqIS3Z0U2A3tiyLyfEtBcby3Jx/8BSFj7U+K85E4=
github.com/minio/mc v0.0.0-20220407151251-dbc09a8bf054/go.mod h1:PIQHcb4uOctKyL/y78tVvC0PCSk6q9UavbSkIAB2/FQ=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=