From 74040b457b50417b58eae7cb17c63428a0e2dd44 Mon Sep 17 00:00:00 2001 From: Poorna Date: Fri, 24 Mar 2023 14:41:23 -0700 Subject: [PATCH] Allow setting sync mode for site replication (#16876) --- cmd/site-replication.go | 91 +++++++++++++++++++++++++---------------- go.mod | 2 +- go.sum | 6 ++- 3 files changed, 60 insertions(+), 39 deletions(-) diff --git a/cmd/site-replication.go b/cmd/site-replication.go index b840823ce..73c9ead52 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -896,6 +896,9 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context, AccessKey: creds.AccessKey, SecretKey: creds.SecretKey, } + if !peer.SyncState.Empty() { + targetToUpdate.ReplicationSync = (peer.SyncState == madmin.SyncEnabled) + } err := globalBucketTargetSys.SetTarget(ctx, bucket, &targetToUpdate, true) if err != nil { return c.annotatePeerErr(peer.Name, "Bucket target update error", err) @@ -927,7 +930,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context, API: "s3v4", Type: madmin.ReplicationService, Region: "", - ReplicationSync: false, + ReplicationSync: peer.SyncState == madmin.SyncEnabled, } var exists bool // true if ARN already exists bucketTarget.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &bucketTarget, peer.DeploymentID) @@ -3443,12 +3446,17 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe ) for _, v := range sites.Sites { - if peer.Endpoint == v.Endpoint { - return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID)) - } if peer.DeploymentID == v.DeploymentID { found = true - if peer.Endpoint == v.Endpoint { + if !peer.SyncState.Empty() { + if globalDeploymentID == peer.DeploymentID { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("invalid sync setting for endpoint %s (deployment id %s)", v.Endpoint, v.DeploymentID)) + } + if peer.Endpoint == "" { // peer.Endpoint may be "" if only sync state is being updated + break + } + } + if peer.Endpoint == v.Endpoint && peer.SyncState.Empty() { return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID)) } admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint) @@ -3469,44 +3477,52 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe if !found { return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("%s not found in existing replicated sites", peer.DeploymentID)) } - - errs := make(map[string]error, len(c.state.Peers)) - var wg sync.WaitGroup - pi := c.state.Peers[peer.DeploymentID] prevPeerInfo := pi - pi.Endpoint = peer.Endpoint + if !peer.SyncState.Empty() { // update replication to peer to be sync/async + pi.SyncState = peer.SyncState + c.state.Peers[peer.DeploymentID] = pi + } + if peer.Endpoint != "" { // `admin replicate update` requested an endpoint change + pi.Endpoint = peer.Endpoint + } - for i, v := range sites.Sites { - if v.DeploymentID == globalDeploymentID { - c.state.Peers[peer.DeploymentID] = pi - continue - } - wg.Add(1) - go func(pi madmin.PeerInfo, i int) { - defer wg.Done() - v := sites.Sites[i] - admClient, err := c.getAdminClient(ctx, v.DeploymentID) - if v.DeploymentID == peer.DeploymentID { - admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint) + if admClient != nil { + errs := make(map[string]error, len(c.state.Peers)) + var wg sync.WaitGroup + + for i, v := range sites.Sites { + if v.DeploymentID == globalDeploymentID { + c.state.Peers[peer.DeploymentID] = pi + continue } + wg.Add(1) + go func(pi madmin.PeerInfo, i int) { + defer wg.Done() + v := sites.Sites[i] + admClient, err := c.getAdminClient(ctx, v.DeploymentID) + if v.DeploymentID == peer.DeploymentID { + admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint) + } + if err != nil { + errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) + return + } + if err = admClient.SRPeerEdit(ctx, pi); err != nil { + errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", v.Name, err)) + return + } + }(pi, i) + } + + wg.Wait() + for dID, err := range errs { if err != nil { - errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) - return + return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err)) } - if err = admClient.SRPeerEdit(ctx, pi); err != nil { - errs[v.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", v.Name, err)) - return - } - }(pi, i) - } - - wg.Wait() - for dID, err := range errs { - if err != nil { - return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err)) } } + // we can now save the cluster replication configuration state. if err = c.saveToDisk(ctx, c.state); err != nil { return madmin.ReplicateEditStatus{ @@ -3514,7 +3530,7 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err), }, nil } - if err = c.updateTargetEndpoints(ctx, prevPeerInfo, peer); err != nil { + if err = c.updateTargetEndpoints(ctx, prevPeerInfo, pi); err != nil { return madmin.ReplicateEditStatus{ Status: madmin.ReplicateAddStatusPartial, ErrDetail: fmt.Sprintf("unable to update peer targets on local: %v", err), @@ -3556,6 +3572,9 @@ func (c *SiteReplicationSys) updateTargetEndpoints(ctx context.Context, prevInfo bucketTarget := target bucketTarget.Secure = ep.Scheme == "https" bucketTarget.Endpoint = ep.Host + if !peer.SyncState.Empty() { + bucketTarget.ReplicationSync = (peer.SyncState == madmin.SyncEnabled) + } err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, true) if err != nil { logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Bucket target creation error", err)) diff --git a/go.mod b/go.mod index 5463bd967..c616a780f 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/minio/dperf v0.4.2 github.com/minio/highwayhash v1.0.2 github.com/minio/kes-go v0.1.0 - github.com/minio/madmin-go/v2 v2.0.16 + github.com/minio/madmin-go/v2 v2.0.17 github.com/minio/minio-go/v7 v7.0.49 github.com/minio/mux v1.9.0 github.com/minio/pkg v1.6.5-0.20230318001333-39b6e90c1c88 diff --git a/go.sum b/go.sum index e25cd9455..b1af803c6 100644 --- a/go.sum +++ b/go.sum @@ -773,8 +773,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT github.com/minio/kes-go v0.1.0 h1:h201DyOYP5sTqajkxFGxmXz/kPbT8HQNX1uh3Yx2PFc= github.com/minio/kes-go v0.1.0/go.mod h1:VorHLaIYis9/MxAHAtXN4d8PUMNKhIxTIlvFt0hBOEo= github.com/minio/madmin-go v1.6.6/go.mod h1:ATvkBOLiP3av4D++2v1UEHC/QzsGtgXD5kYvvRYzdKs= -github.com/minio/madmin-go/v2 v2.0.16 h1:Sj3y17LR+YcWIta9eYZAxSDBtVC1s/tzBkB76v+ynF8= -github.com/minio/madmin-go/v2 v2.0.16/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE= +github.com/minio/madmin-go/v2 v2.0.17 h1:oLg51zMuSl9K8Ng3eO5a/ELTfVEmaS8X5nzQaLwfLO0= +github.com/minio/madmin-go/v2 v2.0.17/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE= github.com/minio/mc v0.0.0-20230318234318-e290a426a131 h1:kgsp58vk1DvNhHK798ep2+yXh3sh34fhjqwG2mZ5ObU= github.com/minio/mc v0.0.0-20230318234318-e290a426a131/go.mod h1:5DTP70Lenpg2TUGOY7I6CWuYcdKGwadNtGhniCSnvlw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= @@ -896,6 +896,8 @@ github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6kt github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/poornas/madmin-go/v2 v2.0.0-20230322192724-66f8756c816f h1:l0vXMhCfsn8kXtI0GbbUO5Aahg1tVVRaJk3n8OOA5SA= +github.com/poornas/madmin-go/v2 v2.0.0-20230322192724-66f8756c816f/go.mod h1:8bL1RMNkblIENFSgGYjeHrzUx9PxROb7OqfNuMU9ivE= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=