Add support for minio tier type (#14468)

cmd/tier.go

@@ -46,7 +46,8 @@ var (
 const (
 	tierConfigFile    = "tier-config.bin"
 	tierConfigFormat  = 1
-	tierConfigVersion = 1
+	tierConfigV1      = 1
+	tierConfigVersion = 2
 
 	minioHotTier = "STANDARD"
 )
@@ -204,6 +205,12 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
 			return errTierInsufficientCreds
 		}
 		cfg.GCS.Creds = base64.URLEncoding.EncodeToString(creds.CredsJSON)
+	case madmin.MinIO:
+		if creds.AccessKey == "" || creds.SecretKey == "" {
+			return errTierInsufficientCreds
+		}
+		cfg.MinIO.AccessKey = creds.AccessKey
+		cfg.MinIO.SecretKey = creds.SecretKey
 	}
 
 	d, err := newWarmBackend(ctx, cfg)
@@ -375,22 +382,22 @@ func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, er
 	}
 
 	// Read header
-	switch binary.LittleEndian.Uint16(data[0:2]) {
+	switch format := binary.LittleEndian.Uint16(data[0:2]); format {
 	case tierConfigFormat:
 	default:
-		return nil, fmt.Errorf("tierConfigInit: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
-	}
-	switch binary.LittleEndian.Uint16(data[2:4]) {
-	case tierConfigVersion:
-	default:
-		return nil, fmt.Errorf("tierConfigInit: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		return nil, fmt.Errorf("tierConfigInit: unknown format: %d", format)
 	}
 
 	cfg := NewTierConfigMgr()
-	_, decErr := cfg.UnmarshalMsg(data[4:])
-	if decErr != nil {
-		return nil, decErr
+	switch version := binary.LittleEndian.Uint16(data[2:4]); version {
+	case tierConfigV1, tierConfigVersion:
+		if _, decErr := cfg.UnmarshalMsg(data[4:]); decErr != nil {
+			return nil, decErr
+		}
+	default:
+		return nil, fmt.Errorf("tierConfigInit: unknown version: %d", version)
 	}
+
 	return cfg, nil
 }
 

cmd/warm-backend-minio.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/minio/madmin-go"
+	minio "github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+type warmBackendMinIO struct {
+	warmBackendS3
+}
+
+var _ WarmBackend = (*warmBackendMinIO)(nil)
+
+func newWarmBackendMinIO(conf madmin.TierMinIO) (*warmBackendMinIO, error) {
+	u, err := url.Parse(conf.Endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
+
+	getRemoteTierTargetInstanceTransportOnce.Do(func() {
+		getRemoteTierTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
+	})
+	opts := &minio.Options{
+		Creds:     creds,
+		Secure:    u.Scheme == "https",
+		Transport: getRemoteTierTargetInstanceTransport,
+	}
+	client, err := minio.New(u.Host, opts)
+	if err != nil {
+		return nil, err
+	}
+	core, err := minio.NewCore(u.Host, opts)
+	if err != nil {
+		return nil, err
+	}
+	return &warmBackendMinIO{
+		warmBackendS3{
+			client: client,
+			core:   core,
+			Bucket: conf.Bucket,
+			Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator),
+		},
+	}, nil
+}

cmd/warm-backend.go

@@ -141,6 +141,8 @@ func newWarmBackend(ctx context.Context, tier madmin.TierConfig) (d WarmBackend,
 		d, err = newWarmBackendAzure(*tier.Azure)
 	case madmin.GCS:
 		d, err = newWarmBackendGCS(*tier.GCS)
+	case madmin.MinIO:
+		d, err = newWarmBackendMinIO(*tier.MinIO)
 	default:
 		return nil, errTierTypeUnsupported
 	}

go.mod

@@ -50,7 +50,7 @@ require (
 	github.com/minio/dperf v0.3.4
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.19.0
-	github.com/minio/madmin-go v1.3.5
+	github.com/minio/madmin-go v1.3.6
 	github.com/minio/minio-go/v7 v7.0.23
 	github.com/minio/parquet-go v1.1.0
 	github.com/minio/pkg v1.1.20
@@ -118,6 +118,7 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
+	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
@@ -193,6 +194,7 @@ require (
 	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
+	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
 	github.com/tidwall/gjson v1.14.0 // indirect
@@ -207,13 +209,14 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
 	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
+	go.uber.org/goleak v1.1.12 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
-	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
+	golang.org/x/tools v0.1.8 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect