From f199afcd6c375d59dcd3bc3e3bca8e28b84487ab Mon Sep 17 00:00:00 2001 From: Poorna Krishnamoorthy Date: Fri, 4 Jun 2021 12:47:00 -0700 Subject: [PATCH] tiering: add aws role support for s3 (#12424) Signed-off-by: Poorna Krishnamoorthy --- cmd/common-main.go | 12 +++++++++++- cmd/tier.go | 12 ++++++++---- cmd/warm-backend-s3.go | 7 ++++++- docs/bucket/lifecycle/README.md | 5 +++++ go.mod | 2 +- go.sum | 6 ++---- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/cmd/common-main.go b/cmd/common-main.go index 01b2c13fb..70ce27c41 100644 --- a/cmd/common-main.go +++ b/cmd/common-main.go @@ -27,6 +27,7 @@ import ( "fmt" "math/rand" "net" + "net/http" "net/url" "os" "path/filepath" @@ -39,6 +40,7 @@ import ( dns2 "github.com/miekg/dns" "github.com/minio/cli" "github.com/minio/kes" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio/internal/auth" "github.com/minio/minio/internal/config" @@ -54,6 +56,7 @@ import ( // serverDebugLog will enable debug printing var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn +var defaultAWSCredProvider []credentials.Provider func init() { rand.Seed(time.Now().UTC().UnixNano()) @@ -74,7 +77,6 @@ func init() { // safe to assume a higher timeout upto 10 minutes. globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf) } - initGlobalContext() globalForwarder = handlers.NewForwarder(&handlers.Forwarder{ @@ -92,6 +94,14 @@ func init() { console.SetColor("Debug", color.New()) gob.Register(StorageErr("")) + + defaultAWSCredProvider = []credentials.Provider{ + &credentials.IAM{ + Client: &http.Client{ + Transport: NewGatewayHTTPTransport(), + }, + }, + } } func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) { diff --git a/cmd/tier.go b/cmd/tier.go index af797da65..da037d1cb 100644 --- a/cmd/tier.go +++ b/cmd/tier.go @@ -144,12 +144,16 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma newCfg := config.Tiers[tierName] switch tierType { case madmin.S3: - if creds.AccessKey == "" || creds.SecretKey == "" { + if (creds.AccessKey == "" || creds.SecretKey == "") && !creds.AWSRole { return errTierInsufficientCreds } - newCfg.S3.AccessKey = creds.AccessKey - newCfg.S3.SecretKey = creds.SecretKey - + switch { + case creds.AWSRole: + newCfg.S3.AWSRole = true + default: + newCfg.S3.AccessKey = creds.AccessKey + newCfg.S3.SecretKey = creds.SecretKey + } case madmin.Azure: if creds.AccessKey == "" || creds.SecretKey == "" { return errTierInsufficientCreds diff --git a/cmd/warm-backend-s3.go b/cmd/warm-backend-s3.go index a5c9ca830..dae1e03cd 100644 --- a/cmd/warm-backend-s3.go +++ b/cmd/warm-backend-s3.go @@ -106,7 +106,12 @@ func newWarmBackendS3(conf madmin.TierS3) (*warmBackendS3, error) { if err != nil { return nil, err } - creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "") + var creds *credentials.Credentials + if conf.AWSRole { + creds = credentials.NewChainCredentials(defaultAWSCredProvider) + } else { + creds = credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "") + } getRemoteTargetInstanceTransportOnce.Do(func() { getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute) }) diff --git a/docs/bucket/lifecycle/README.md b/docs/bucket/lifecycle/README.md index ca911d28f..4446717ba 100644 --- a/docs/bucket/lifecycle/README.md +++ b/docs/bucket/lifecycle/README.md @@ -116,6 +116,11 @@ Using above tier, set up a lifecycle rule with transition: mc ilm add --expiry-days 365 --transition-days 45 --storage-class "AZURETIER" myminio/srcbucket ``` +Note: In the case of S3, it is possible to create a tier from MinIO running in EC2 to S3 using AWS role attached to EC2 as credentials instead of accesskey/secretkey: +``` +mc admin tier add s3 source S3TIER --bucket s3bucket --prefix testprefix/ --use-aws-role +``` + Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized. ``` diff --git a/go.mod b/go.mod index 3d945df89..76965176b 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/minio/csvparser v1.0.0 github.com/minio/highwayhash v1.0.2 github.com/minio/kes v0.14.0 - github.com/minio/madmin-go v1.0.9 + github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 github.com/minio/parquet-go v1.0.0 github.com/minio/pkg v1.0.4 diff --git a/go.sum b/go.sum index 39e0d4c6f..e92993364 100644 --- a/go.sum +++ b/go.sum @@ -483,8 +483,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q= github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA= -github.com/minio/madmin-go v1.0.9 h1:zXZMppnqboIyELPirHcU6qxrnJkVwj2k7rLIB0T12sY= -github.com/minio/madmin-go v1.0.9/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs= +github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4 h1:AxtnO3AODg0t2IPXbrqmDBhGZTcrUhlT/ixdLQQ3164= +github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo= @@ -492,8 +492,6 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWky github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw= github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiIk= github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA= -github.com/minio/pkg v1.0.3 h1:tUhM6lG/BdNB0+5f2RbE4ifCAYwMs6cRJnZ/AY0WIeQ= -github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8= github.com/minio/pkg v1.0.4 h1:+BmaCENP6BaMm9PsGK6L1L5MKulWDxl4qobvJYf6m/E= github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8= github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=