self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-07-08 19:56:05 +00:00
Code Issues Projects Releases Wiki Activity

tiering: add aws role support for s3 (#12424)

Browse Source 
Signed-off-by: Poorna Krishnamoorthy <poorna@minio.io>
This commit is contained in:
Poorna Krishnamoorthy 2021-06-04 12:47:00 -07:00 committed by GitHub
parent 36b2f6d11d
commit f199afcd6c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 33 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/common-main.go
View File

@ -27,6 +27,7 @@ import (
"fmt"
"math/rand"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
@ -39,6 +40,7 @@ import (
dns2 "github.com/miekg/dns"
"github.com/minio/cli"
"github.com/minio/kes"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
@ -54,6 +56,7 @@ import (
// serverDebugLog will enable debug printing
var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
var defaultAWSCredProvider []credentials.Provider
func init() {
rand.Seed(time.Now().UTC().UnixNano())
@ -74,7 +77,6 @@ func init() {
// safe to assume a higher timeout upto 10 minutes.
globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
}
initGlobalContext()
globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
@ -92,6 +94,14 @@ func init() {
console.SetColor("Debug", color.New())
gob.Register(StorageErr(""))
defaultAWSCredProvider = []credentials.Provider{
&credentials.IAM{
Client: &http.Client{
Transport: NewGatewayHTTPTransport(),
},
},
}
}
func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {

12
cmd/tier.go
View File

@ -144,12 +144,16 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
newCfg := config.Tiers[tierName]
switch tierType {
case madmin.S3:
if creds.AccessKey == "" || creds.SecretKey == "" {
if (creds.AccessKey == "" || creds.SecretKey == "") && !creds.AWSRole {
return errTierInsufficientCreds
}
newCfg.S3.AccessKey = creds.AccessKey
newCfg.S3.SecretKey = creds.SecretKey
switch {
case creds.AWSRole:
newCfg.S3.AWSRole = true
default:
newCfg.S3.AccessKey = creds.AccessKey
newCfg.S3.SecretKey = creds.SecretKey
}
case madmin.Azure:
if creds.AccessKey == "" || creds.SecretKey == "" {
return errTierInsufficientCreds

7
cmd/warm-backend-s3.go
View File

@ -106,7 +106,12 @@ func newWarmBackendS3(conf madmin.TierS3) (*warmBackendS3, error) {
if err != nil {
return nil, err
}
creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
var creds *credentials.Credentials
if conf.AWSRole {
creds = credentials.NewChainCredentials(defaultAWSCredProvider)
} else {
creds = credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
}
getRemoteTargetInstanceTransportOnce.Do(func() {
getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
})

5
docs/bucket/lifecycle/README.md
View File

@ -116,6 +116,11 @@ Using above tier, set up a lifecycle rule with transition:
mc ilm add --expiry-days 365 --transition-days 45 --storage-class "AZURETIER" myminio/srcbucket
```
Note: In the case of S3, it is possible to create a tier from MinIO running in EC2 to S3 using AWS role attached to EC2 as credentials instead of accesskey/secretkey:
```
mc admin tier add s3 source S3TIER --bucket s3bucket --prefix testprefix/ --use-aws-role
```
Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized.
```

2
go.mod
View File

@ -44,7 +44,7 @@ require (
github.com/minio/csvparser v1.0.0
github.com/minio/highwayhash v1.0.2
github.com/minio/kes v0.14.0
github.com/minio/madmin-go v1.0.9
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78
github.com/minio/parquet-go v1.0.0
github.com/minio/pkg v1.0.4

6
go.sum
View File

@ -483,8 +483,8 @@ github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
github.com/minio/madmin-go v1.0.9 h1:zXZMppnqboIyELPirHcU6qxrnJkVwj2k7rLIB0T12sY=
github.com/minio/madmin-go v1.0.9/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4 h1:AxtnO3AODg0t2IPXbrqmDBhGZTcrUhlT/ixdLQQ3164=
github.com/minio/madmin-go v1.0.10-0.20210602195449-b1bf23ec13e4/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
@ -492,8 +492,6 @@ github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78 h1:v7OMbUnWky
github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiIk=
github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
github.com/minio/pkg v1.0.3 h1:tUhM6lG/BdNB0+5f2RbE4ifCAYwMs6cRJnZ/AY0WIeQ=
github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
github.com/minio/pkg v1.0.4 h1:+BmaCENP6BaMm9PsGK6L1L5MKulWDxl4qobvJYf6m/E=
github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=