diff --git a/cmd/tier-handlers.go b/cmd/tier-handlers.go index 856c5ca57..b588753ec 100644 --- a/cmd/tier-handlers.go +++ b/cmd/tier-handlers.go @@ -140,6 +140,7 @@ func (api adminAPIHandlers) ListTierHandler(w http.ResponseWriter, r *http.Reque return } + w.Header().Set(tierCfgRefreshAtHdr, globalTierConfigMgr.refreshedAt().String()) writeSuccessResponseJSON(w, data) } diff --git a/cmd/tier.go b/cmd/tier.go index 05e347219..890dc7026 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -1,4 +1,4 @@ -// Copyright (c) 2015-2023 MinIO, Inc. +// Copyright (c) 2015-2024 MinIO, Inc // // This file is part of MinIO Object Storage stack // @@ -23,6 +23,7 @@ import ( "encoding/base64" "encoding/binary" "fmt" + "math/rand" "net/http" "path" "strings" @@ -33,6 +34,7 @@ import ( "github.com/minio/minio/internal/crypto" "github.com/minio/minio/internal/hash" "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" "github.com/prometheus/client_golang/prometheus" ) @@ -74,12 +76,15 @@ const ( // tierConfigPath refers to remote tier config object name var tierConfigPath = path.Join(minioConfigPrefix, tierConfigFile) +const tierCfgRefreshAtHdr = "X-MinIO-TierCfg-RefreshedAt" + // TierConfigMgr holds the collection of remote tiers configured in this deployment. type TierConfigMgr struct { sync.RWMutex `msg:"-"` drivercache map[string]WarmBackend `msg:"-"` - Tiers map[string]madmin.TierConfig `json:"tiers"` + Tiers map[string]madmin.TierConfig `json:"tiers"` + lastRefreshedAt time.Time `msg:"-"` } type tierMetrics struct { @@ -172,6 +177,12 @@ func (t *tierMetrics) Report() []Metric { return metrics } +func (config *TierConfigMgr) refreshedAt() time.Time { + config.RLock() + defer config.RUnlock() + return config.lastRefreshedAt +} + // IsTierValid returns true if there exists a remote tier by name tierName, // otherwise returns false. func (config *TierConfigMgr) IsTierValid(tierName string) bool { @@ -413,7 +424,7 @@ func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, * return nil, nil, err } if GlobalKMS == nil { - return NewPutObjReader(hr), &ObjectOptions{}, nil + return NewPutObjReader(hr), &ObjectOptions{MaxParity: true}, nil } // Note: Local variables with names ek, oek, etc are named inline with @@ -443,6 +454,7 @@ func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, * opts := &ObjectOptions{ UserDefined: metadata, MTime: UTCNow(), + MaxParity: true, } return pReader, opts, nil @@ -455,6 +467,9 @@ func (config *TierConfigMgr) Reload(ctx context.Context, objAPI ObjectLayer) err case nil: break case errConfigNotFound: // nothing to reload + // To maintain the invariance that lastRefreshedAt records the + // timestamp of last successful refresh + config.lastRefreshedAt = UTCNow() return nil default: return err @@ -474,7 +489,7 @@ func (config *TierConfigMgr) Reload(ctx context.Context, objAPI ObjectLayer) err for tier, cfg := range newConfig.Tiers { config.Tiers[tier] = cfg } - + config.lastRefreshedAt = UTCNow() return nil } @@ -501,6 +516,31 @@ func NewTierConfigMgr() *TierConfigMgr { } } +func (config *TierConfigMgr) refreshTierConfig(ctx context.Context, objAPI ObjectLayer) { + const tierCfgRefresh = 15 * time.Minute + r := rand.New(rand.NewSource(time.Now().UnixNano())) + randInterval := func() time.Duration { + return time.Duration(r.Float64() * 5 * float64(time.Second)) + } + + // To avoid all MinIO nodes reading the tier config object at the same + // time. + t := time.NewTimer(tierCfgRefresh + randInterval()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + err := config.Reload(ctx, objAPI) + if err != nil { + logger.LogIf(ctx, err) + } + } + t.Reset(tierCfgRefresh + randInterval()) + } +} + // loadTierConfig loads remote tier configuration from objAPI. func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, error) { if objAPI == nil { @@ -536,19 +576,11 @@ func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, er return cfg, nil } -// Reset clears remote tier configured and clears tier driver cache. -func (config *TierConfigMgr) Reset() { - config.Lock() - for k := range config.drivercache { - delete(config.drivercache, k) - } - for k := range config.Tiers { - delete(config.Tiers, k) - } - config.Unlock() -} - // Init initializes tier configuration reading from objAPI func (config *TierConfigMgr) Init(ctx context.Context, objAPI ObjectLayer) error { - return config.Reload(ctx, objAPI) + err := config.Reload(ctx, objAPI) + if globalIsDistErasure { + go config.refreshTierConfig(ctx, objAPI) + } + return err }