minio/cmd/admin-handlers-site-replication.go

624 lines
19 KiB
Go
Raw Normal View History

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"encoding/gob"
"encoding/json"
"errors"
"io"
"net/http"
"strings"
"sync/atomic"
"time"
"github.com/dustin/go-humanize"
"github.com/minio/madmin-go/v3"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/mux"
"github.com/minio/pkg/v3/policy"
)
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
if objectAPI == nil {
return
}
var sites []madmin.PeerSite
if err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
opts := getSRAddOptions(r)
status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
body, err := json.Marshal(status)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, body)
}
func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
return
}
2021-12-27 07:10:34 +00:00
// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
//
// used internally to tell current cluster to enable SR with
// the provided peer clusters and service account.
2021-12-27 07:10:34 +00:00
func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
if objectAPI == nil {
return
}
2021-12-27 07:10:34 +00:00
var joinArg madmin.SRPeerJoinReq
if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
2021-12-27 07:10:34 +00:00
// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
if objectAPI == nil {
return
}
vars := mux.Vars(r)
bucket := vars["bucket"]
operation := madmin.BktOp(vars["operation"])
var err error
switch operation {
default:
err = errSRInvalidRequest(errInvalidArgument)
case madmin.MakeWithVersioningBktOp:
createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt")))
if cerr != nil {
createdAt = timeSentinel
}
opts := MakeBucketOptions{
LockEnabled: r.Form.Get("lockEnabled") == "true",
VersioningEnabled: r.Form.Get("versioningEnabled") == "true",
ForceCreate: r.Form.Get("forceCreate") == "true",
CreatedAt: createdAt,
}
err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
case madmin.ConfigureReplBktOp:
err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp:
err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
Force: operation == madmin.ForceDeleteBucketBktOp,
SRDeleteOp: getSRBucketDeleteOp(true),
})
case madmin.PurgeDeletedBucketOp:
globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket)
}
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
2021-12-27 07:10:34 +00:00
// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
if objectAPI == nil {
return
}
var item madmin.SRIAMItem
if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
var err error
switch item.Type {
default:
err = errSRInvalidRequest(errInvalidArgument)
case madmin.SRIAMItemPolicy:
if item.Policy == nil {
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
} else {
2023-09-04 19:57:37 +00:00
policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
if perr != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
return
}
if policy.IsEmpty() {
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
} else {
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt)
}
}
case madmin.SRIAMItemSvcAcc:
err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt)
case madmin.SRIAMItemPolicyMapping:
err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt)
case madmin.SRIAMItemSTSAcc:
err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt)
case madmin.SRIAMItemIAMUser:
err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt)
case madmin.SRIAMItemGroupInfo:
err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt)
}
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
2021-12-27 07:10:34 +00:00
func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
if objectAPI == nil {
return
}
var item madmin.SRBucketMeta
if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if item.Bucket == "" {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
return
}
var err error
switch item.Type {
default:
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
case madmin.SRBucketMetaTypePolicy:
if item.Policy == nil {
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
} else {
2023-09-04 19:57:37 +00:00
bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
if berr != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
return
}
if bktPolicy.IsEmpty() {
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
} else {
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt)
}
}
case madmin.SRBucketMetaTypeQuotaConfig:
if item.Quota == nil {
err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt)
} else {
quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
case madmin.SRBucketMetaTypeVersionConfig:
err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt)
case madmin.SRBucketMetaTypeTags:
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
case madmin.SRBucketMetaTypeObjectLockConfig:
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
case madmin.SRBucketMetaTypeSSEConfig:
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
case madmin.SRBucketMetaLCConfig:
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
}
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
if objectAPI == nil {
return
}
info, err := globalSiteReplicationSys.GetClusterInfo(ctx)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err = json.NewEncoder(w).Encode(info); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
2021-12-27 07:10:34 +00:00
func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
if objectAPI == nil {
return
}
idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx)
if err := json.NewEncoder(w).Encode(idpSettings); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
2022-09-19 18:05:16 +00:00
data, err := io.ReadAll(body)
if err != nil {
return SRError{
Cause: err,
Code: ErrSiteReplicationInvalidRequest,
}
}
if encryptionKey != "" {
data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
if err != nil {
return SRError{
Cause: err,
Code: ErrSiteReplicationInvalidRequest,
}
}
}
return json.Unmarshal(data, v)
}
// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
if objectAPI == nil {
return
}
opts := getSRStatusOptions(r)
// default options to all if status options are unset for backward compatibility
var dfltOpts madmin.SRStatusOptions
if opts == dfltOpts {
opts.Buckets = true
opts.Users = true
opts.Policies = true
opts.Groups = true
opts.ILMExpiryRules = true
}
info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Report the ILMExpiryStats only if at least one site has replication of ILM expiry enabled
var replicateILMExpiry bool
for _, site := range info.Sites {
if site.ReplicateILMExpiry {
replicateILMExpiry = true
break
}
}
if !replicateILMExpiry {
// explicitly send nil for ILMExpiryStats
info.ILMExpiryStats = nil
}
if err = json.NewEncoder(w).Encode(info); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
if objectAPI == nil {
return
}
opts := getSRStatusOptions(r)
info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI, opts)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err = json.NewEncoder(w).Encode(info); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
if objectAPI == nil {
return
}
var site madmin.PeerInfo
err := parseJSONBody(ctx, r.Body, &site, cred.SecretKey)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
opts := getSREditOptions(r)
status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
body, err := json.Marshal(status)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, body)
}
func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
return
}
// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
//
// used internally to tell current cluster to update endpoint for peer
func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
if objectAPI == nil {
return
}
var pi madmin.PeerInfo
if err := parseJSONBody(ctx, r.Body, &pi, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
//
// used internally to tell current cluster to update site replication state
func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
if objectAPI == nil {
return
}
var state madmin.SRStateEditReq
if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
q := r.Form
opts.Buckets = q.Get("buckets") == "true"
opts.Policies = q.Get("policies") == "true"
opts.Groups = q.Get("groups") == "true"
opts.Users = q.Get("users") == "true"
opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
opts.PeerState = q.Get("peer-state") == "true"
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
opts.EntityValue = q.Get("entityvalue")
opts.ShowDeleted = q.Get("showDeleted") == "true"
opts.Metrics = q.Get("metrics") == "true"
return
}
// SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
if objectAPI == nil {
return
}
var rreq madmin.SRRemoveReq
err := parseJSONBody(ctx, r.Body, &rreq, "")
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq)
if err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
body, err := json.Marshal(status)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, body)
}
// SRPeerRemove - PUT /minio/admin/v3/site-replication/peer/remove
//
// used internally to tell current cluster to update endpoint for peer
func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
if objectAPI == nil {
return
}
var req madmin.SRRemoveReq
if err := parseJSONBody(ctx, r.Body, &req, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil {
adminLogIf(ctx, err)
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
2023-09-04 19:57:37 +00:00
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction)
if objectAPI == nil {
return
}
var peerSite madmin.PeerInfo
if err := parseJSONBody(ctx, r.Body, &peerSite, ""); err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
vars := mux.Vars(r)
op := madmin.SiteResyncOp(vars["operation"])
var (
status madmin.SRResyncOpStatus
err error
)
switch op {
case madmin.SiteResyncStart:
status, err = globalSiteReplicationSys.startResync(ctx, objectAPI, peerSite)
case madmin.SiteResyncCancel:
status, err = globalSiteReplicationSys.cancelResync(ctx, objectAPI, peerSite)
default:
err = errSRInvalidRequest(errInvalidArgument)
}
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
body, err := json.Marshal(status)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, body)
}
// SiteReplicationDevNull - everything goes to io.Discard
// [POST] /minio/admin/v3/site-replication/devnull
func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
Send AuditLog via new middleware fn for admin APIs (#17632) A new middleware function is added for admin handlers, including options for modifying certain behaviors. This admin middleware: - sets the handler context via reflection in the request and sends AuditLog - checks for object API availability (skipping it if a flag is passed) - enables gzip compression (skipping it if a flag is passed) - enables header tracing (adding body tracing if a flag is passed) While the new function is a middleware, due to the flags used for conditional behavior modification, which is used in each route registration call. To try to ensure that no regressions are introduced, the following changes were done mechanically mostly with `sed` and regexp: - Remove defer logger.AuditLog in admin handlers - Replace newContext() calls with r.Context() - Update admin routes registration calls Bonus: remove unused NetSpeedtestHandler Since the new adminMiddleware function checks for object layer presence by default, we need to pass the `noObjLayerFlag` explicitly to admin handlers that should work even when it is not available. The following admin handlers do not require it: - ServerInfoHandler - StartProfilingHandler - DownloadProfilingHandler - ProfileHandler - SiteReplicationDevNull - SiteReplicationNetPerf - TraceHandler For these handlers adminMiddleware does not check for the object layer presence (disabled by passing the `noObjLayerFlag`), and for all other handlers, the pre-check ensures that the handler is not called when the object layer is not available - the client would get a ErrServerNotInitialized and can retry later. This `noObjLayerFlag` is added based on existing behavior for these handlers only.
2023-07-13 21:52:21 +00:00
ctx := r.Context()
globalSiteNetPerfRX.Connect()
defer globalSiteNetPerfRX.Disconnect()
connectTime := time.Now()
for {
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
// would mean the network is not stable. Logging here will help in debugging network issues.
if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
adminLogIf(ctx, err)
}
}
if err != nil {
if errors.Is(err, io.EOF) {
w.WriteHeader(http.StatusNoContent)
} else {
w.WriteHeader(http.StatusBadRequest)
}
break
}
}
}
// SiteReplicationNetPerf - everything goes to io.Discard
// [POST] /minio/admin/v3/site-replication/netperf
func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
durationStr := r.Form.Get(peerRESTDuration)
duration, _ := time.ParseDuration(durationStr)
if duration < globalNetPerfMinDuration {
duration = globalNetPerfMinDuration
}
result := siteNetperf(r.Context(), duration)
adminLogIf(r.Context(), gob.NewEncoder(w).Encode(result))
}