From 9a34fd5c4a1e1e9f3de050f49c00edea8e32b4b5 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 19 Nov 2020 18:43:58 -0800 Subject: [PATCH] Revert "Revert "Add delete marker replication support (#10396)"" This reverts commit 267d7bf0a9f114065314a0b2863f7fcc9e923012. --- cmd/api-datatypes.go | 13 + cmd/bucket-handlers.go | 25 +- cmd/bucket-replication.go | 157 +++++++++++- cmd/data-crawler.go | 35 +++ cmd/erasure-metadata.go | 5 +- cmd/erasure-object.go | 130 +++++++--- cmd/http/headers.go | 6 + cmd/metacache-entries.go | 2 +- cmd/object-api-datatypes.go | 3 +- cmd/object-api-interface.go | 21 +- cmd/object-api-options.go | 61 ++++- cmd/object-handlers.go | 40 ++- cmd/storage-datatypes.go | 33 +++ cmd/storage-datatypes_gen.go | 128 +++++++++- cmd/web-handlers.go | 74 +++++- cmd/xl-storage-format-utils.go | 4 +- cmd/xl-storage-format-v2.go | 74 +++++- cmd/xl-storage-format-v2_gen.go | 230 +++++++++++------- cmd/xl-storage.go | 6 +- .../replication/DELETE_bucket-replication.png | Bin 0 -> 37993 bytes docs/bucket/replication/README.md | 29 ++- go.mod | 2 +- go.sum | 28 +-- pkg/bucket/replication/replication.go | 15 +- pkg/bucket/replication/rule.go | 51 +++- 25 files changed, 950 insertions(+), 222 deletions(-) create mode 100644 docs/bucket/replication/DELETE_bucket-replication.png diff --git a/cmd/api-datatypes.go b/cmd/api-datatypes.go index e663b44ab..6303ed3bd 100644 --- a/cmd/api-datatypes.go +++ b/cmd/api-datatypes.go @@ -18,6 +18,7 @@ package cmd import ( "encoding/xml" + "time" ) // DeletedObject objects deleted @@ -26,12 +27,24 @@ type DeletedObject struct { DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"` ObjectName string `xml:"Key,omitempty"` VersionID string `xml:"VersionId,omitempty"` + // Replication status of DeleteMarker + DeleteMarkerReplicationStatus string + // MTime of DeleteMarker on source that needs to be propagated to replica + DeleteMarkerMTime time.Time + // Status of versioned delete (of object or DeleteMarker) + VersionPurgeStatus VersionPurgeStatusType } // ObjectToDelete carries key name for the object to delete. type ObjectToDelete struct { ObjectName string `xml:"Key"` VersionID string `xml:"VersionId"` + // Replication status of DeleteMarker + DeleteMarkerReplicationStatus string + // Status of versioned delete (of object or DeleteMarker) + VersionPurgeStatus VersionPurgeStatusType + // Version ID of delete marker + DeleteMarkerVersionID string } // createBucketConfiguration container for bucket configuration request from client. diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 7e33986a7..330936908 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -404,6 +404,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, if api.CacheAPI() != nil { getObjectInfoFn = api.CacheAPI().GetObjectInfo } + replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects) dErrs := make([]DeleteError, len(deleteObjects.Objects)) for index, object := range deleteObjects.Objects { @@ -439,6 +440,18 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // Avoid duplicate objects, we use map to filter them out. if _, ok := objectsToDelete[object]; !ok { + if replicateDeletes { + if delMarker, replicate := checkReplicateDelete(ctx, getObjectInfoFn, bucket, ObjectToDelete{ObjectName: object.ObjectName, VersionID: object.VersionID}); replicate { + if object.VersionID != "" { + object.VersionPurgeStatus = Pending + if delMarker { + object.DeleteMarkerVersionID = object.VersionID + } + } else { + object.DeleteMarkerReplicationStatus = string(replication.Pending) + } + } + } objectsToDelete[object] = index } } @@ -467,6 +480,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, }] apiErr := toAPIError(ctx, errs[i]) if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" { + dObjects[i].DeleteMarkerReplicationStatus = deleteList[i].DeleteMarkerReplicationStatus + dObjects[i].VersionPurgeStatus = deleteList[i].VersionPurgeStatus deletedObjects[dindex] = dObjects[i] continue } @@ -491,7 +506,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, // Write success response. writeSuccessResponseXML(w, encodedSuccessResponse) - + for _, dobj := range deletedObjects { + if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending { + globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{ + DeletedObject: dobj, + Bucket: bucket, + }) + } + } // Notify deleted event for objects. for _, dobj := range deletedObjects { eventName := event.ObjectRemovedDelete @@ -1298,7 +1320,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } - if err = globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, configData); err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go index 16b329697..728df5e0d 100644 --- a/cmd/bucket-replication.go +++ b/cmd/bucket-replication.go @@ -83,7 +83,7 @@ func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object strin if permErr != ErrNone { return false } - return mustReplicater(ctx, r, bucket, object, meta, replStatus) + return mustReplicater(ctx, bucket, object, meta, replStatus) } // mustReplicate returns true if object meets replication criteria. @@ -91,11 +91,11 @@ func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone { return false } - return mustReplicater(ctx, r, bucket, object, meta, replStatus) + return mustReplicater(ctx, bucket, object, meta, replStatus) } // mustReplicater returns true if object meets replication criteria. -func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool { +func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) bool { if globalIsGateway { return false } @@ -120,6 +120,127 @@ func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, return cfg.Replicate(opts) } +// returns true if any of the objects being deleted qualifies for replication. +func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool { + c, err := getReplicationConfig(ctx, bucket) + if err != nil || c == nil { + return false + } + for _, obj := range objects { + if c.HasActiveRules(obj.ObjectName, true) { + return true + } + } + return false +} + +// returns whether object version is a deletemarker and if object qualifies for replication +func checkReplicateDelete(ctx context.Context, getObjectInfoFn GetObjectInfoFn, bucket string, dobj ObjectToDelete) (dm, replicate bool) { + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil || rcfg == nil { + return false, false + } + oi, err := getObjectInfoFn(ctx, bucket, dobj.ObjectName, ObjectOptions{VersionID: dobj.VersionID}) + // when incoming delete is removal of a delete marker( a.k.a versioned delete), + // GetObjectInfo returns extra information even though it returns errFileNotFound + if err != nil { + validReplStatus := false + switch oi.ReplicationStatus { + case replication.Pending, replication.Complete, replication.Failed: + validReplStatus = true + } + if oi.DeleteMarker && validReplStatus { + return oi.DeleteMarker, true + } + return oi.DeleteMarker, false + } + opts := replication.ObjectOpts{ + Name: dobj.ObjectName, + SSEC: crypto.SSEC.IsEncrypted(oi.UserDefined), + UserTags: oi.UserTags, + DeleteMarker: true, + VersionID: dobj.VersionID, + } + return oi.DeleteMarker, rcfg.Replicate(opts) +} + +// replicate deletes to the designated replication target if replication configuration +// has delete marker replication or delete replication (MinIO extension to allow deletes where version id +// is specified) enabled. +// Similar to bucket replication for PUT operation, soft delete (a.k.a setting delete marker) and +// permanent deletes (by specifying a version ID in the delete operation) have three states "Pending", "Complete" +// and "Failed" to mark the status of the replication of "DELETE" operation. All failed operations can +// then be retried by healing. In the case of permanent deletes, until the replication is completed on the +// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently +// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds +// on target. +func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) { + bucket := dobj.Bucket + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil || rcfg == nil { + return + } + tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn) + if tgt == nil { + return + } + versionID := dobj.DeleteMarkerVersionID + if versionID == "" { + versionID = dobj.VersionID + } + rmErr := tgt.RemoveObject(ctx, rcfg.GetDestination().Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{ + VersionID: versionID, + Internal: miniogo.AdvancedRemoveOptions{ + ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "", + ReplicationMTime: dobj.DeleteMarkerMTime, + ReplicationStatus: miniogo.ReplicationStatusReplica, + }, + }) + + replicationStatus := dobj.DeleteMarkerReplicationStatus + versionPurgeStatus := dobj.VersionPurgeStatus + + if rmErr != nil { + if dobj.VersionID == "" { + replicationStatus = string(replication.Failed) + } else { + versionPurgeStatus = Failed + } + } else { + if dobj.VersionID == "" { + replicationStatus = string(replication.Complete) + } else { + versionPurgeStatus = Complete + } + } + if replicationStatus == string(replication.Failed) || versionPurgeStatus == Failed { + objInfo := ObjectInfo{ + Name: dobj.ObjectName, + DeleteMarker: dobj.DeleteMarker, + VersionID: versionID, + ReplicationStatus: replication.StatusType(dobj.DeleteMarkerReplicationStatus), + } + eventArg := &eventArgs{ + BucketName: bucket, + Object: objInfo, + Host: "Internal: [Replication]", + EventName: event.ObjectReplicationFailed, + } + sendEvent(*eventArg) + } + // Update metadata on the delete marker or purge permanent delete if replication success. + if _, err = objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{ + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + DeleteMarkerReplicationStatus: replicationStatus, + Versioned: globalBucketVersioningSys.Enabled(bucket), + VersionPurgeStatus: versionPurgeStatus, + VersionSuspended: globalBucketVersioningSys.Suspended(bucket), + }); err != nil { + logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err)) + } +} + func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) { meta := make(map[string]string) for k, v := range objInfo.UserDefined { @@ -304,18 +425,37 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri return dst } +// DeletedObjectVersionInfo has info on deleted object +type DeletedObjectVersionInfo struct { + DeletedObject + Bucket string +} type replicationState struct { // add future metrics here - replicaCh chan ObjectInfo + replicaCh chan ObjectInfo + replicaDeleteCh chan DeletedObjectVersionInfo } func (r *replicationState) queueReplicaTask(oi ObjectInfo) { + if r == nil { + return + } select { case r.replicaCh <- oi: default: } } +func (r *replicationState) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) { + if r == nil { + return + } + select { + case r.replicaDeleteCh <- doi: + default: + } +} + var ( globalReplicationState *replicationState // TODO: currently keeping it conservative @@ -332,11 +472,13 @@ func newReplicationState() *replicationState { globalReplicationConcurrent = 1 } rs := &replicationState{ - replicaCh: make(chan ObjectInfo, 10000), + replicaCh: make(chan ObjectInfo, 10000), + replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000), } go func() { <-GlobalContext.Done() close(rs.replicaCh) + close(rs.replicaDeleteCh) }() return rs } @@ -354,6 +496,11 @@ func (r *replicationState) addWorker(ctx context.Context, objectAPI ObjectLayer) return } replicateObject(ctx, oi, objectAPI) + case doi, ok := <-r.replicaDeleteCh: + if !ok { + return + } + replicateDelete(ctx, doi, objectAPI) } } }() diff --git a/cmd/data-crawler.go b/cmd/data-crawler.go index 976de4879..7c80ff50a 100644 --- a/cmd/data-crawler.go +++ b/cmd/data-crawler.go @@ -792,8 +792,43 @@ func sleepDuration(d time.Duration, x float64) { // healReplication will heal a scanned item that has failed replication. func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) { + if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() { + //heal delete marker replication failure or versioned delete replication failure + if meta.oi.ReplicationStatus == replication.Pending || + meta.oi.ReplicationStatus == replication.Failed || + meta.oi.VersionPurgeStatus == Failed || meta.oi.VersionPurgeStatus == Pending { + i.healReplicationDeletes(ctx, o, meta) + return + } + } if meta.oi.ReplicationStatus == replication.Pending || meta.oi.ReplicationStatus == replication.Failed { globalReplicationState.queueReplicaTask(meta.oi) } } + +// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes. +func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, meta actionMeta) { + // handle soft delete and permanent delete failures here. + if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() { + versionID := "" + dmVersionID := "" + if meta.oi.VersionPurgeStatus.Empty() { + dmVersionID = meta.oi.VersionID + } else { + versionID = meta.oi.VersionID + } + globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{ + DeletedObject: DeletedObject{ + ObjectName: meta.oi.Name, + DeleteMarkerVersionID: dmVersionID, + VersionID: versionID, + DeleteMarkerReplicationStatus: string(meta.oi.ReplicationStatus), + DeleteMarkerMTime: meta.oi.ModTime, + DeleteMarker: meta.oi.DeleteMarker, + VersionPurgeStatus: meta.oi.VersionPurgeStatus, + }, + Bucket: meta.oi.Bucket, + }) + } +} diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go index 386852b16..1aa9526fd 100644 --- a/cmd/erasure-metadata.go +++ b/cmd/erasure-metadata.go @@ -139,7 +139,9 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo { // Add replication status to the object info objInfo.ReplicationStatus = replication.StatusType(fi.Metadata[xhttp.AmzBucketReplicationStatus]) - + if fi.Deleted { + objInfo.ReplicationStatus = replication.StatusType(fi.DeleteMarkerReplicationStatus) + } // etag/md5Sum has already been extracted. We need to // remove to avoid it from appearing as part of // response headers. e.g, X-Minio-* or X-Amz-*. @@ -155,6 +157,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string) ObjectInfo { } else { objInfo.StorageClass = globalMinioDefaultStorageClass } + objInfo.VersionPurgeStatus = fi.VersionPurgeStatus // Success. return objInfo } diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index a3776c4a5..d7732e989 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -28,6 +28,7 @@ import ( "github.com/minio/minio-go/v7/pkg/tags" xhttp "github.com/minio/minio/cmd/http" "github.com/minio/minio/cmd/logger" + "github.com/minio/minio/pkg/bucket/replication" "github.com/minio/minio/pkg/mimedb" "github.com/minio/minio/pkg/sync/errgroup" ) @@ -404,7 +405,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin if fi.Deleted { objInfo = fi.ToObjectInfo(bucket, object) - if opts.VersionID == "" { + if opts.VersionID == "" || opts.DeleteMarker { return objInfo, toObjectErr(errFileNotFound, bucket, object) } // Make sure to return object info to provide extra information. @@ -737,7 +738,6 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object return disks[index].DeleteVersion(ctx, bucket, object, fi) }, index) } - // return errors if any during deletion return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum) } @@ -803,26 +803,35 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec versions := make([]FileInfo, len(objects)) for i := range objects { + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + uuid := opts.VersionID + if uuid == "" { + uuid = mustGetUUID() + } + if objects[i].VersionID == "" { - if opts.Versioned || opts.VersionSuspended { - fi := FileInfo{ - Name: objects[i].ObjectName, - ModTime: UTCNow(), - Deleted: true, // delete marker + if (opts.Versioned || opts.VersionSuspended) && !HasSuffix(objects[i].ObjectName, SlashSeparator) { + versions[i] = FileInfo{ + Name: objects[i].ObjectName, + ModTime: modTime, + Deleted: true, // delete marker + DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus, + VersionPurgeStatus: objects[i].VersionPurgeStatus, } if opts.Versioned { - fi.VersionID = mustGetUUID() + versions[i].VersionID = uuid } - // versioning suspended means we add `null` - // version as delete marker - - versions[i] = fi continue } } versions[i] = FileInfo{ - Name: objects[i].ObjectName, - VersionID: objects[i].VersionID, + Name: objects[i].ObjectName, + VersionID: objects[i].VersionID, + DeleteMarkerReplicationStatus: objects[i].DeleteMarkerReplicationStatus, + VersionPurgeStatus: objects[i].VersionPurgeStatus, } } @@ -869,14 +878,19 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec ObjectPathUpdated(pathJoin(bucket, objects[objIndex].ObjectName)) if versions[objIndex].Deleted { dobjects[objIndex] = DeletedObject{ - DeleteMarker: versions[objIndex].Deleted, - DeleteMarkerVersionID: versions[objIndex].VersionID, - ObjectName: decodeDirObject(versions[objIndex].Name), + DeleteMarker: versions[objIndex].Deleted, + DeleteMarkerVersionID: versions[objIndex].VersionID, + DeleteMarkerMTime: versions[objIndex].ModTime, + DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus, + ObjectName: versions[objIndex].Name, + VersionPurgeStatus: versions[objIndex].VersionPurgeStatus, } } else { dobjects[objIndex] = DeletedObject{ - ObjectName: decodeDirObject(versions[objIndex].Name), - VersionID: versions[objIndex].VersionID, + ObjectName: versions[objIndex].Name, + VersionID: versions[objIndex].VersionID, + VersionPurgeStatus: versions[objIndex].VersionPurgeStatus, + DeleteMarkerReplicationStatus: versions[objIndex].DeleteMarkerReplicationStatus, } } } @@ -906,16 +920,20 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec // any error as it is not necessary for the handler to reply back a // response to the client request. func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { - defer ObjectPathUpdated(path.Join(bucket, object)) + versionFound := true goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts) if gerr != nil && goi.Name == "" { switch gerr.(type) { case InsufficientReadQuorum: return objInfo, InsufficientWriteQuorum{} } - return objInfo, gerr + // For delete marker replication, versionID being replicated will not exist on disk + if opts.DeleteMarker { + versionFound = false + } else { + return objInfo, gerr + } } - // Acquire a write lock before deleting the object. lk := er.NewNSLock(bucket, object) if err = lk.GetLock(ctx, globalDeleteOperationTimeout); err != nil { @@ -925,23 +943,58 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string storageDisks := er.getDisks() writeQuorum := len(storageDisks)/2 + 1 + var markDelete bool + // Determine whether to mark object deleted for replication + if goi.VersionID != "" { + markDelete = true + } + // Default deleteMarker to true if object is under versioning + deleteMarker := true + if gerr == nil { + deleteMarker = goi.VersionID != "" + } + if opts.VersionID != "" { + // case where replica version needs to be deleted on target cluster + if versionFound && opts.DeleteMarkerReplicationStatus == replication.Replica.String() { + markDelete = false + } + if opts.VersionPurgeStatus.Empty() && opts.DeleteMarkerReplicationStatus == "" { + markDelete = false + } + if opts.DeleteMarker && opts.VersionPurgeStatus == Complete { + markDelete = false + } + // determine if the version represents an object delete + // deleteMarker = true + if versionFound && !goi.DeleteMarker { // implies a versioned delete of object + deleteMarker = false + } + } - if opts.VersionID == "" { - if opts.Versioned || opts.VersionSuspended { + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + if markDelete { + if (opts.Versioned || opts.VersionSuspended) && !HasSuffix(object, SlashSeparator) { fi := FileInfo{ - Name: object, - Deleted: true, - ModTime: UTCNow(), + Name: object, + Deleted: deleteMarker, + MarkDeleted: markDelete, + ModTime: modTime, + DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus, + VersionPurgeStatus: opts.VersionPurgeStatus, } - if opts.Versioned { fi.VersionID = mustGetUUID() + if opts.VersionID != "" { + fi.VersionID = opts.VersionID + } } - // versioning suspended means we add `null` // version as delete marker - // Add delete marker, since we don't have any version specified explicitly. + // Or if a particular version id needs to be replicated. if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil { return objInfo, toObjectErr(err, bucket, object) } @@ -951,8 +1004,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string // Delete the object version on all disks. if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{ - Name: object, - VersionID: opts.VersionID, + Name: object, + VersionID: opts.VersionID, + MarkDeleted: markDelete, + Deleted: deleteMarker, + ModTime: modTime, + DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus, + VersionPurgeStatus: opts.VersionPurgeStatus, }); err != nil { return objInfo, toObjectErr(err, bucket, object) } @@ -964,7 +1022,13 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string } } - return ObjectInfo{Bucket: bucket, Name: decodeDirObject(object), VersionID: opts.VersionID}, nil + return ObjectInfo{ + Bucket: bucket, + Name: object, + VersionID: opts.VersionID, + VersionPurgeStatus: opts.VersionPurgeStatus, + ReplicationStatus: replication.StatusType(opts.DeleteMarkerReplicationStatus), + }, nil } // Send the successful but partial upload/delete, however ignore diff --git a/cmd/http/headers.go b/cmd/http/headers.go index 24a20c15a..f6fb9592c 100644 --- a/cmd/http/headers.go +++ b/cmd/http/headers.go @@ -132,6 +132,12 @@ const ( // Reports number of drives currently healing MinIOHealingDrives = "x-minio-healing-drives" + + // Header indicates if the delete marker should be preserved by client + MinIOSourceDeleteMarker = "x-minio-source-deletemarker" + + // Header indicates if the delete marker version needs to be purged. + MinIOSourceDeleteMarkerDelete = "x-minio-source-deletemarker-delete" ) // Common http query params S3 API diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go index 2481e8716..39e31f18e 100644 --- a/cmd/metacache-entries.go +++ b/cmd/metacache-entries.go @@ -151,7 +151,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro }, }, nil } - return getFileInfoVersions(e.metadata, bucket, e.name) + return getFileInfoVersions(e.metadata, bucket, e.name, false) } // metaCacheEntries is a slice of metacache entries. diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go index 57d4428bc..8d3e583ba 100644 --- a/cmd/object-api-datatypes.go +++ b/cmd/object-api-datatypes.go @@ -207,7 +207,8 @@ type ObjectInfo struct { Legacy bool // indicates object on disk is in legacy data format // backendType indicates which backend filled this structure - backendType BackendType + backendType BackendType + VersionPurgeStatus VersionPurgeStatusType } // MultipartInfo captures metadata information about the uploadId diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index ddb11a737..bb0b8a203 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -36,15 +36,18 @@ type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts Objec // ObjectOptions represents object options for ObjectLayer object operations type ObjectOptions struct { - ServerSideEncryption encrypt.ServerSide - VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended. - Versioned bool // indicates if the bucket is versioned - WalkVersions bool // indicates if the we are interested in walking versions - VersionID string // Specifies the versionID which needs to be overwritten or read - MTime time.Time // Is only set in POST/PUT operations - UserDefined map[string]string // only set in case of POST/PUT operations - PartNumber int // only useful in case of GetObject/HeadObject - CheckPrecondFn CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation + ServerSideEncryption encrypt.ServerSide + VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended. + Versioned bool // indicates if the bucket is versioned + WalkVersions bool // indicates if the we are interested in walking versions + VersionID string // Specifies the versionID which needs to be overwritten or read + MTime time.Time // Is only set in POST/PUT operations + DeleteMarker bool // Is only set in DELETE operations for delete marker replication + UserDefined map[string]string // only set in case of POST/PUT operations + PartNumber int // only useful in case of GetObject/HeadObject + CheckPrecondFn CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation + DeleteMarkerReplicationStatus string // Is only set in DELETE operations + VersionPurgeStatus VersionPurgeStatusType // Is only set in DELETE operations for delete marker version to be permanently deleted. } // BucketOptions represents bucket options for ObjectLayer bucket operations diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index ee715b84e..e2b6b511a 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -123,6 +123,20 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec } opts.PartNumber = partNumber opts.VersionID = vid + delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker)) + if delMarker != "" { + if delMarker != "true" && delMarker != "false" { + logger.LogIf(ctx, err) + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false")), + } + } + if delMarker == "true" { + opts.DeleteMarker = true + } + } return opts, nil } @@ -134,6 +148,49 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts } opts.Versioned = versioned opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket) + delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker)) + if delMarker != "" { + if delMarker != "true" && delMarker != "false" { + logger.LogIf(ctx, err) + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false")), + } + } + if delMarker == "true" { + opts.DeleteMarker = true + } + } + + purgeVersion := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarkerDelete)) + if purgeVersion != "" { + if purgeVersion != "true" && purgeVersion != "false" { + logger.LogIf(ctx, err) + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarkerDelete, fmt.Errorf("DeleteMarkerPurge should be true or false")), + } + } + if purgeVersion == "true" { + opts.VersionPurgeStatus = Complete + } + } + + mtime := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime)) + if mtime != "" { + opts.MTime, err = time.Parse(time.RFC3339, mtime) + if err != nil { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err), + } + } + } else { + opts.MTime = UTCNow() + } return opts, nil } @@ -160,7 +217,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada } } mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime)) - var mtime time.Time + mtime := UTCNow() if mtimeStr != "" { mtime, err = time.Parse(time.RFC3339, mtimeStr) if err != nil { @@ -170,8 +227,6 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err), } } - } else { - mtime = UTCNow() } etag := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceETag)) if etag != "" { diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index e84ab763a..e0ce3909b 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -2705,6 +2705,23 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) return } + _, replicateDel := checkReplicateDelete(ctx, getObjectInfo, bucket, ObjectToDelete{ObjectName: object, VersionID: opts.VersionID}) + if replicateDel { + if opts.VersionID != "" { + opts.VersionPurgeStatus = Pending + } else { + opts.DeleteMarkerReplicationStatus = string(replication.Pending) + } + } + + if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() { + // check if replica has permission to be deleted. + if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object); apiErrCode != ErrNone { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) + return + } + opts.DeleteMarkerReplicationStatus = replication.Replica.String() + } apiErr := ErrNone if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled { @@ -2736,8 +2753,29 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http. } // Ignore delete object errors while replying to client, since we are suppposed to reply only 204. } - setPutObjHeaders(w, objInfo, true) + if replicateDel { + dmVersionID := "" + versionID := "" + if objInfo.DeleteMarker { + dmVersionID = objInfo.VersionID + } else { + versionID = objInfo.VersionID + } + globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{ + DeletedObject: DeletedObject{ + ObjectName: object, + VersionID: versionID, + DeleteMarkerVersionID: dmVersionID, + DeleteMarkerReplicationStatus: string(objInfo.ReplicationStatus), + DeleteMarkerMTime: objInfo.ModTime, + DeleteMarker: objInfo.DeleteMarker, + VersionPurgeStatus: objInfo.VersionPurgeStatus, + }, + Bucket: bucket, + }) + } + setPutObjHeaders(w, objInfo, true) writeSuccessNoContent(w) } diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index a78575c4a..b75f0d71f 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -136,6 +136,39 @@ type FileInfo struct { // Erasure info for all objects. Erasure ErasureInfo + + // DeleteMarkerReplicationStatus is set when this FileInfo represents + // replication on a DeleteMarker + MarkDeleted bool // mark this version as deleted + DeleteMarkerReplicationStatus string + VersionPurgeStatus VersionPurgeStatusType +} + +// VersionPurgeStatusKey denotes purge status in metadata +const VersionPurgeStatusKey = "purgestatus" + +// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication +type VersionPurgeStatusType string + +const ( + // Pending - versioned delete replication is pending. + Pending VersionPurgeStatusType = "PENDING" + + // Complete - versioned delete replication is now complete, erase version on disk. + Complete VersionPurgeStatusType = "COMPLETE" + + // Failed - versioned delete replication failed. + Failed VersionPurgeStatusType = "FAILED" +) + +// Empty returns true if purge status was not set. +func (v VersionPurgeStatusType) Empty() bool { + return string(v) == "" +} + +// Pending returns true if the version is pending purge. +func (v VersionPurgeStatusType) Pending() bool { + return v == Pending || v == Failed } // newFileInfo - initializes new FileInfo, allocates a fresh erasure info. diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index a4608092e..4dfaa7771 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -342,8 +342,8 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err) return } - if zb0001 != 13 { - err = msgp.ArrayError{Wanted: 13, Got: zb0001} + if zb0001 != 16 { + err = msgp.ArrayError{Wanted: 16, Got: zb0001} return } z.Volume, err = dc.ReadString() @@ -448,13 +448,32 @@ func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "Erasure") return } + z.MarkDeleted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + z.DeleteMarkerReplicationStatus, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkerReplicationStatus") + return + } + { + var zb0004 string + zb0004, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + z.VersionPurgeStatus = VersionPurgeStatusType(zb0004) + } return } // EncodeMsg implements msgp.Encodable func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { - // array header, size 13 - err = en.Append(0x9d) + // array header, size 16 + err = en.Append(0xdc, 0x0, 0x10) if err != nil { return } @@ -542,14 +561,29 @@ func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Erasure") return } + err = en.WriteBool(z.MarkDeleted) + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + err = en.WriteString(z.DeleteMarkerReplicationStatus) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkerReplicationStatus") + return + } + err = en.WriteString(string(z.VersionPurgeStatus)) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } return } // MarshalMsg implements msgp.Marshaler func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // array header, size 13 - o = append(o, 0x9d) + // array header, size 16 + o = append(o, 0xdc, 0x0, 0x10) o = msgp.AppendString(o, z.Volume) o = msgp.AppendString(o, z.Name) o = msgp.AppendString(o, z.VersionID) @@ -578,6 +612,9 @@ func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { err = msgp.WrapError(err, "Erasure") return } + o = msgp.AppendBool(o, z.MarkDeleted) + o = msgp.AppendString(o, z.DeleteMarkerReplicationStatus) + o = msgp.AppendString(o, string(z.VersionPurgeStatus)) return } @@ -589,8 +626,8 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err) return } - if zb0001 != 13 { - err = msgp.ArrayError{Wanted: 13, Got: zb0001} + if zb0001 != 16 { + err = msgp.ArrayError{Wanted: 16, Got: zb0001} return } z.Volume, bts, err = msgp.ReadStringBytes(bts) @@ -695,13 +732,32 @@ func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Erasure") return } + z.MarkDeleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + z.DeleteMarkerReplicationStatus, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkerReplicationStatus") + return + } + { + var zb0004 string + zb0004, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + z.VersionPurgeStatus = VersionPurgeStatusType(zb0004) + } o = bts return } // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *FileInfo) Msgsize() (s int) { - s = 1 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize + s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.MapHeaderSize if z.Metadata != nil { for za0001, za0002 := range z.Metadata { _ = za0002 @@ -712,7 +768,7 @@ func (z *FileInfo) Msgsize() (s int) { for za0003 := range z.Parts { s += z.Parts[za0003].Msgsize() } - s += z.Erasure.Msgsize() + s += z.Erasure.Msgsize() + msgp.BoolSize + msgp.StringPrefixSize + len(z.DeleteMarkerReplicationStatus) + msgp.StringPrefixSize + len(string(z.VersionPurgeStatus)) return } @@ -1281,6 +1337,58 @@ func (z *FilesInfoVersions) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *VersionPurgeStatusType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionPurgeStatusType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VersionPurgeStatusType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VersionPurgeStatusType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VersionPurgeStatusType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionPurgeStatusType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VersionPurgeStatusType) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + // DecodeMsg implements msgp.Decodable func (z *VolInfo) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte diff --git a/cmd/web-handlers.go b/cmd/web-handlers.go index 2e3ba3640..a6cd5d0da 100644 --- a/cmd/web-handlers.go +++ b/cmd/web-handlers.go @@ -612,6 +612,10 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, if web.CacheAPI() != nil { deleteObjects = web.CacheAPI().DeleteObjects } + getObjectInfoFn := objectAPI.GetObjectInfo + if web.CacheAPI() != nil { + getObjectInfoFn = web.CacheAPI().GetObjectInfo + } claims, owner, authErr := webRequestAuthenticate(r) if authErr != nil { @@ -714,8 +718,27 @@ next: return toJSONError(ctx, errAccessDenied) } } + _, replicateDel := checkReplicateDelete(ctx, getObjectInfoFn, args.BucketName, ObjectToDelete{ObjectName: objectName}) + if replicateDel { + opts.DeleteMarkerReplicationStatus = string(replication.Pending) + opts.DeleteMarker = true + } + + oi, err := deleteObject(ctx, objectAPI, web.CacheAPI(), args.BucketName, objectName, nil, r, opts) + if replicateDel { + globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{ + DeletedObject: DeletedObject{ + ObjectName: objectName, + DeleteMarkerVersionID: oi.VersionID, + DeleteMarkerReplicationStatus: string(oi.ReplicationStatus), + DeleteMarkerMTime: oi.ModTime, + DeleteMarker: oi.DeleteMarker, + VersionPurgeStatus: oi.VersionPurgeStatus, + }, + Bucket: args.BucketName, + }) + } - _, err = deleteObject(ctx, objectAPI, web.CacheAPI(), args.BucketName, objectName, nil, r, opts) logger.LogIf(ctx, err) continue } @@ -760,9 +783,40 @@ next: // Reached maximum delete requests, attempt a delete for now. break } - objects = append(objects, ObjectToDelete{ - ObjectName: obj.Name, - }) + if obj.ReplicationStatus == replication.Replica { + if authErr == errNoAuthToken { + // Check if object is allowed to be deleted anonymously + if !globalPolicySys.IsAllowed(policy.Args{ + Action: iampolicy.ReplicateDeleteAction, + BucketName: args.BucketName, + ConditionValues: getConditionValues(r, "", "", nil), + IsOwner: false, + ObjectName: objectName, + }) { + return toJSONError(ctx, errAccessDenied) + } + } else { + if !globalIAMSys.IsAllowed(iampolicy.Args{ + AccountName: claims.AccessKey, + Action: iampolicy.ReplicateDeleteAction, + BucketName: args.BucketName, + ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()), + IsOwner: owner, + ObjectName: objectName, + Claims: claims.Map(), + }) { + return toJSONError(ctx, errAccessDenied) + } + } + } + // since versioned delete is not available on web browser, yet - this is a simple DeleteMarker replication + _, replicateDel := checkReplicateDelete(ctx, getObjectInfoFn, args.BucketName, ObjectToDelete{ObjectName: obj.Name}) + objToDel := ObjectToDelete{ObjectName: obj.Name} + if replicateDel { + objToDel.DeleteMarkerReplicationStatus = string(replication.Pending) + } + + objects = append(objects, objToDel) } // Nothing to do. @@ -772,7 +826,11 @@ next: // Deletes a list of objects. deletedObjects, errs := deleteObjects(ctx, args.BucketName, objects, opts) - for _, err := range errs { + for i, err := range errs { + if err != nil && !isErrObjectNotFound(err) { + deletedObjects[i].DeleteMarkerReplicationStatus = objects[i].DeleteMarkerReplicationStatus + deletedObjects[i].VersionPurgeStatus = objects[i].VersionPurgeStatus + } if err != nil { logger.LogIf(ctx, err) break next @@ -799,6 +857,12 @@ next: UserAgent: r.UserAgent(), Host: handlers.GetSourceIP(r), }) + if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending { + globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{ + DeletedObject: dobj, + Bucket: args.BucketName, + }) + } } } } diff --git a/cmd/xl-storage-format-utils.go b/cmd/xl-storage-format-utils.go index c24a78b91..98ebffb95 100644 --- a/cmd/xl-storage-format-utils.go +++ b/cmd/xl-storage-format-utils.go @@ -34,13 +34,13 @@ func (v versionsSorter) Less(i, j int) bool { return v[i].ModTime.After(v[j].ModTime) } -func getFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVersions, error) { +func getFileInfoVersions(xlMetaBuf []byte, volume, path string, showPendingDeletes bool) (FileInfoVersions, error) { if isXL2V1Format(xlMetaBuf) { var xlMeta xlMetaV2 if err := xlMeta.Load(xlMetaBuf); err != nil { return FileInfoVersions{}, err } - versions, latestModTime, err := xlMeta.ListVersions(volume, path) + versions, latestModTime, err := xlMeta.ListVersions(volume, path, showPendingDeletes) if err != nil { return FileInfoVersions{}, err } diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 4f697109e..f832feae6 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -139,8 +139,9 @@ func (e ChecksumAlgo) valid() bool { // xlMetaV2DeleteMarker defines the data struct for the delete marker journal type type xlMetaV2DeleteMarker struct { - VersionID [16]byte `json:"ID" msg:"ID"` // Version ID for delete marker - ModTime int64 `json:"MTime" msg:"MTime"` // Object delete marker modified time + VersionID [16]byte `json:"ID" msg:"ID"` // Version ID for delete marker + ModTime int64 `json:"MTime" msg:"MTime"` // Object delete marker modified time + MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Delete marker internal metadata } // xlMetaV2Object defines the data struct for object journal type @@ -354,11 +355,13 @@ func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) versionID = uuid.UUID(j.VersionID).String() } fi := FileInfo{ - Volume: volume, - Name: path, - ModTime: time.Unix(0, j.ModTime).UTC(), - VersionID: versionID, - Deleted: true, + Volume: volume, + Name: path, + ModTime: time.Unix(0, j.ModTime).UTC(), + VersionID: versionID, + Deleted: true, + DeleteMarkerReplicationStatus: string(j.MetaSys[xhttp.AmzBucketReplicationStatus]), + VersionPurgeStatus: VersionPurgeStatusType(string(j.MetaSys[VersionPurgeStatusKey])), } return fi, nil } @@ -408,6 +411,9 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) { fi.Metadata[k] = string(v) } + if strings.EqualFold(k, VersionPurgeStatusKey) { + fi.VersionPurgeStatus = VersionPurgeStatusType(string(v)) + } } fi.Erasure.Algorithm = j.ErasureAlgorithm.String() fi.Erasure.Index = j.ErasureIndex @@ -446,12 +452,36 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { DeleteMarker: &xlMetaV2DeleteMarker{ VersionID: uv, ModTime: fi.ModTime.UnixNano(), + MetaSys: make(map[string][]byte), }, } if !ventry.Valid() { return "", false, errors.New("internal error: invalid version entry generated") } } + updateVersion := false + if fi.VersionPurgeStatus.Empty() && (fi.DeleteMarkerReplicationStatus == "REPLICA" || fi.DeleteMarkerReplicationStatus == "") { + updateVersion = fi.MarkDeleted + } else { + // for replication scenario + if fi.Deleted && fi.VersionPurgeStatus != Complete { + if !fi.VersionPurgeStatus.Empty() || fi.DeleteMarkerReplicationStatus != "" { + updateVersion = true + } + } + // object or delete-marker versioned delete is not complete + if !fi.VersionPurgeStatus.Empty() && fi.VersionPurgeStatus != Complete { + updateVersion = true + } + } + if fi.Deleted { + if fi.DeleteMarkerReplicationStatus != "" { + ventry.DeleteMarker.MetaSys[xhttp.AmzBucketReplicationStatus] = []byte(fi.DeleteMarkerReplicationStatus) + } + if !fi.VersionPurgeStatus.Empty() { + ventry.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) + } + } for i, version := range z.Versions { if !version.Valid() { @@ -468,12 +498,28 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { } case DeleteType: if bytes.Equal(version.DeleteMarker.VersionID[:], uv[:]) { - z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) - if fi.Deleted { - z.Versions = append(z.Versions, ventry) + if updateVersion { + delete(z.Versions[i].DeleteMarker.MetaSys, xhttp.AmzBucketReplicationStatus) + delete(z.Versions[i].DeleteMarker.MetaSys, VersionPurgeStatusKey) + if fi.DeleteMarkerReplicationStatus != "" { + z.Versions[i].DeleteMarker.MetaSys[xhttp.AmzBucketReplicationStatus] = []byte(fi.DeleteMarkerReplicationStatus) + } + if !fi.VersionPurgeStatus.Empty() { + z.Versions[i].DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) + } + } else { + z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) + if fi.MarkDeleted && (fi.VersionPurgeStatus.Empty() || (fi.VersionPurgeStatus != Complete)) { + z.Versions = append(z.Versions, ventry) + } } return "", len(z.Versions) == 0, nil } + case ObjectType: + if bytes.Equal(version.ObjectV2.VersionID[:], uv[:]) && updateVersion { + z.Versions[i].ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.VersionPurgeStatus) + return "", len(z.Versions) == 0, nil + } } } @@ -518,7 +564,6 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { z.Versions = append(z.Versions, ventry) return "", false, nil } - return "", false, errFileVersionNotFound } @@ -538,7 +583,9 @@ func (z xlMetaV2) TotalSize() int64 { // ListVersions lists current versions, and current deleted // versions returns error for unexpected entries. -func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTime time.Time, err error) { +// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted +// but waiting to be replicated +func (z xlMetaV2) ListVersions(volume, path string, showPendingDeletes bool) (versions []FileInfo, modTime time.Time, err error) { var latestModTime time.Time var latestVersionID string for _, version := range z.Versions { @@ -551,6 +598,9 @@ func (z xlMetaV2) ListVersions(volume, path string) (versions []FileInfo, modTim fi, err = version.ObjectV2.ToFileInfo(volume, path) case DeleteType: fi, err = version.DeleteMarker.ToFileInfo(volume, path) + if !fi.VersionPurgeStatus.Empty() && !showPendingDeletes { + continue + } case LegacyType: fi, err = version.ObjectV1.ToFileInfo(volume, path) } diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go index bc604eb81..307147ca1 100644 --- a/cmd/xl-storage-format-v2_gen.go +++ b/cmd/xl-storage-format-v2_gen.go @@ -338,6 +338,36 @@ func (z *xlMetaV2DeleteMarker) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "ModTime") return } + case "MetaSys": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0002) + } else if len(z.MetaSys) > 0 { + for key := range z.MetaSys { + delete(z.MetaSys, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0002 string + var za0003 []byte + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0003, err = dc.ReadBytes(za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + z.MetaSys[za0002] = za0003 + } default: err = dc.Skip() if err != nil { @@ -351,9 +381,23 @@ func (z *xlMetaV2DeleteMarker) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 2 + // omitempty: check for empty values + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 3 bits */ + if z.MetaSys == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } // write "ID" - err = en.Append(0x82, 0xa2, 0x49, 0x44) + err = en.Append(0xa2, 0x49, 0x44) if err != nil { return } @@ -372,19 +416,63 @@ func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "ModTime") return } + if (zb0001Mask & 0x4) == 0 { // if not empty + // write "MetaSys" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.MetaSys))) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + for za0002, za0003 := range z.MetaSys { + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + err = en.WriteBytes(za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + } + } return } // MarshalMsg implements msgp.Marshaler func (z *xlMetaV2DeleteMarker) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 2 + // omitempty: check for empty values + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 3 bits */ + if z.MetaSys == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if zb0001Len == 0 { + return + } // string "ID" - o = append(o, 0x82, 0xa2, 0x49, 0x44) + o = append(o, 0xa2, 0x49, 0x44) o = msgp.AppendBytes(o, (z.VersionID)[:]) // string "MTime" o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendInt64(o, z.ModTime) + if (zb0001Mask & 0x4) == 0 { // if not empty + // string "MetaSys" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys))) + for za0002, za0003 := range z.MetaSys { + o = msgp.AppendString(o, za0002) + o = msgp.AppendBytes(o, za0003) + } + } return } @@ -418,6 +506,36 @@ func (z *xlMetaV2DeleteMarker) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "ModTime") return } + case "MetaSys": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0002) + } else if len(z.MetaSys) > 0 { + for key := range z.MetaSys { + delete(z.MetaSys, key) + } + } + for zb0002 > 0 { + var za0002 string + var za0003 []byte + zb0002-- + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0003, bts, err = msgp.ReadBytesBytes(bts, za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + z.MetaSys[za0002] = za0003 + } default: bts, err = msgp.Skip(bts) if err != nil { @@ -432,7 +550,13 @@ func (z *xlMetaV2DeleteMarker) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *xlMetaV2DeleteMarker) Msgsize() (s int) { - s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 6 + msgp.Int64Size + s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize + if z.MetaSys != nil { + for za0002, za0003 := range z.MetaSys { + _ = za0003 + s += msgp.StringPrefixSize + len(za0002) + msgp.BytesPrefixSize + len(za0003) + } + } return } @@ -1409,40 +1533,11 @@ func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) { if z.DeleteMarker == nil { z.DeleteMarker = new(xlMetaV2DeleteMarker) } - var zb0003 uint32 - zb0003, err = dc.ReadMapHeader() + err = z.DeleteMarker.DecodeMsg(dc) if err != nil { err = msgp.WrapError(err, "DeleteMarker") return } - for zb0003 > 0 { - zb0003-- - field, err = dc.ReadMapKeyPtr() - if err != nil { - err = msgp.WrapError(err, "DeleteMarker") - return - } - switch msgp.UnsafeString(field) { - case "ID": - err = dc.ReadExactBytes((z.DeleteMarker.VersionID)[:]) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "VersionID") - return - } - case "MTime": - z.DeleteMarker.ModTime, err = dc.ReadInt64() - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "ModTime") - return - } - default: - err = dc.Skip() - if err != nil { - err = msgp.WrapError(err, "DeleteMarker") - return - } - } - } } default: err = dc.Skip() @@ -1540,25 +1635,9 @@ func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { return } } else { - // map header, size 2 - // write "ID" - err = en.Append(0x82, 0xa2, 0x49, 0x44) + err = z.DeleteMarker.EncodeMsg(en) if err != nil { - return - } - err = en.WriteBytes((z.DeleteMarker.VersionID)[:]) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "VersionID") - return - } - // write "MTime" - err = en.Append(0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) - if err != nil { - return - } - err = en.WriteInt64(z.DeleteMarker.ModTime) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "ModTime") + err = msgp.WrapError(err, "DeleteMarker") return } } @@ -1624,13 +1703,11 @@ func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { if z.DeleteMarker == nil { o = msgp.AppendNil(o) } else { - // map header, size 2 - // string "ID" - o = append(o, 0x82, 0xa2, 0x49, 0x44) - o = msgp.AppendBytes(o, (z.DeleteMarker.VersionID)[:]) - // string "MTime" - o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) - o = msgp.AppendInt64(o, z.DeleteMarker.ModTime) + o, err = z.DeleteMarker.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } } } return @@ -1709,40 +1786,11 @@ func (z *xlMetaV2Version) UnmarshalMsg(bts []byte) (o []byte, err error) { if z.DeleteMarker == nil { z.DeleteMarker = new(xlMetaV2DeleteMarker) } - var zb0003 uint32 - zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + bts, err = z.DeleteMarker.UnmarshalMsg(bts) if err != nil { err = msgp.WrapError(err, "DeleteMarker") return } - for zb0003 > 0 { - zb0003-- - field, bts, err = msgp.ReadMapKeyZC(bts) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker") - return - } - switch msgp.UnsafeString(field) { - case "ID": - bts, err = msgp.ReadExactBytes(bts, (z.DeleteMarker.VersionID)[:]) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "VersionID") - return - } - case "MTime": - z.DeleteMarker.ModTime, bts, err = msgp.ReadInt64Bytes(bts) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker", "ModTime") - return - } - default: - bts, err = msgp.Skip(bts) - if err != nil { - err = msgp.WrapError(err, "DeleteMarker") - return - } - } - } } default: bts, err = msgp.Skip(bts) @@ -1774,7 +1822,7 @@ func (z *xlMetaV2Version) Msgsize() (s int) { if z.DeleteMarker == nil { s += msgp.NilSize } else { - s += 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 6 + msgp.Int64Size + s += z.DeleteMarker.Msgsize() } return } diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index e8967b6fc..e98666f0a 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -379,7 +379,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac // Remove filename which is the meta file. item.transformMetaDir() - fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath()) + fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), true) if err != nil { return 0, errSkipFile } @@ -423,7 +423,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac } totalSize += size } - item.healReplication(ctx, objAPI, actionMeta{oi: oi}) + item.healReplication(ctx, objAPI, actionMeta{oi: version.ToObjectInfo(item.bucket, item.objectPath())}) } return totalSize, nil }) @@ -897,7 +897,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st continue } - fiv, err = getFileInfoVersions(xlMetaBuf, volume, walkResult.entry) + fiv, err = getFileInfoVersions(xlMetaBuf, volume, walkResult.entry, false) if err != nil { continue } diff --git a/docs/bucket/replication/DELETE_bucket-replication.png b/docs/bucket/replication/DELETE_bucket-replication.png new file mode 100644 index 0000000000000000000000000000000000000000..19cdebe6d56d080bda38c4853e01128b546946cc GIT binary patch literal 37993 zcmeFZgb& zVP?KHd++;wp7;Cygzxq^wuA7CYh7_(=XtHg4%JjwAiGX?orHvhOi5Ap1qsQeDiV^5 zx38WD-x+ddp$8w=-YM$4kdSb*5dS;pnFnzvA-PAQBrC1snZ7l5&NbmEgX2t7>uGBx zE%dK7<)`XnvU%m~u07T52{NR+;>w2X>UFgl(*4&^42TYl)fU(Gh!4w_0$xrk!N;so^Cr73*X}ncWv=2O zSuXhjO7+KHnLgcz2qa;bfYlK3*fF97wn{}yE?8axUwjng72Bsa@oe=0{~)>AsU|!p z;I=V(92aw&L)~|UKWl&E0{Fpw&!aIt*io@5F5l`)Ny*~`Wv}&fBqZ-CjC}Xohx9l2 ziGB!F;rs8$e_in3EckCB{1*%U2M+CWCYY`&SX-~|%r%K)YiepT#N3K1D=Xn}c)qE* zxw)EJ-zDSlwR7YQA`}t^`uh5Qez=e1p4R7{de4rMe75}2;Ii7&)064FQOM)JA6E8A zv*;#+$l&mBrO)wB>p*!$MMYuZ+mJ;>1if&Xag+OByWFq7y`CsDPDn_Y*Cx4>ltw3| z)x7tEbix$o_b4GYoP=#>v{0KKyPR%i^~Lk|_YdjN(skjklU?v#Qfh8{JG)-rJUCqP z6tksJTwV@CPgzH>oa3Yt7JUBz+%4TjyZusES4dD$P)KNeV!}Gh-_URdEY8rtAZOvv zVI+Zy+lYpYii?YDqC|gWbTl5*tzDpAQ(bM5KSxJT@3B3-+;{nVS3}{sJ4qH7IsHUg zx>iHA2j#jgJT{-B^B8`0yc0m(FerU(X+cd!5a_%xK2<%ibqtvW|@A6Syp z%gRFShn+!kb8%rY12BKw(L__f)WZe>9-HO0lCl<93q|@P{f>>j*Yg6>k$Fibrzr!z zE1$W_Lh$2;(3S+_taL4X{f5`AA8TuCf#2>jNGB#FjE#;qAB`Ihog9HHpN3p~9Y_qq z&SZsop-xeA^K4m*)Z5REBXR|`epqyVe*XPTD{bu&^!7|H1Ug&K3u|>3I2@mz78|_E zH#|8>gS+puEJSi6AdT+(SKzz6ha(wfg~&@8=4jLIU1wq00D+$u-jgomUw&GIqCKaq z3DQJx!svF!#AZ3dL|fY>dsXB>VGTbV$N3owlcIw8r*?F;%{ZsXEi8ENV(hK=X0~>AhV}2>pxy(kiX#8{0Y&Vs zzb;068!+leGM+Z>fG~d|rOp&{dvt2JGEPDx3Zgp1PIQ8mMr+QE2f_8v{}gH$>@*+t z|GIiGWu0|2e*X0mh=zO`$Qo&LNQ?W%Xd!rnJdBM_T31)s+wR;`XGdF-+`f4MscRcs zTg@w}b~%UMATxqRAc@-_neO{s1obrq!V_+N@qG+3Rvv%V(N;=iOYK5@Wol}wYEnHY zqMM7V{4l%LkC#>%ZmJgp$v8c}DE{cXT)JEH zE#mwi8N98~&a6?MrGZGp$ z;G_10_3w{>c}422qoc{U*R359^i`c<81qhmY8@RNpeha%DLmZWt?<4eLLbR5k;tur z45wpdk)ohNZEkJNw)l8hBW#wa)gs4GcJ}tw!uLt`!6lkTx+6p$+S*-`Um;8ezM|mn zzWV?7kFrn`pZ5$fj457nmFp1NY&AXlucJ1w#sq*vrRrIMPWkW0zb;sO{bTU4QT@C2 z&v!wA7N}>?^+?~`e{<*naBnYI?A`D0Ps@h8qE^3uxbV|Iy359hfh1^uW?ro{L|E`x zkh3hbj6=%aGRP5>wvDZ=L6sGNy@({z_pHv^7tB=pu1fJ$S62%N2!OvmTSyee08HwF z0E9Y$`nZd0wjg26NTZ`*2HDT5s-mJ|X=&-=a_B|g1UN!vr8B3sa2qpy_A5q>S?-%s z$+M%M+x^JoW%$lN#b^4Vr!J9FEird|xNyaNcRs_^zS(K9;}Y&NMzQw0*;^sfTkmEZaO%o9jAsTEOMD{)qxM$(#yZ}jP46>3)N&*

~P`b4M2Kti^*wiuRv&rDC7x!G2TW!4)rydSJD zyqkoj3lIGNf_Tut|d!e(V?@tD;FW%!lxDpd^Y~TZc-32YnVEX@lNb0Tk zZIb|{`m^aSoQu{33-wi$CyC82E!_q=FcmyKIk{Hz&+}fzgp;s-|F=`6FACIMmZAjR z(TRojTu9SDP0k%)A+t5e32p5-;0X&SkQItobl;y4GwUDbbP)9p;IoOyKi+%Nef9eZ zXrO;CG5!RtCg_D*GXVWo?3uNIkkCy@ePI;KGmo8F8is+c_Knvh%Ask~pO`eX>dRXd zfti%~g@uI;4PKm`>4^IaQXN)q9wm_+j?`D~sXE%WFTgLeWDi`GGSdMhPFR>r^zvU9 z>kH5AXIovoM+;ijc!|C?@Yd1!d+Wek#MQM;fF=f_zp=SVPfs6Q?@2?l&<$ctv{HoT zOYV%~o&7gq2b#IAsY83GGR(fHo0spsbF^g<2v1AJ(%ngt{4du0-*C(Bz5U^@FC*)j z^Yf$K8n^Pq*)Sn#gv^4+^r5bc+1l2<91V{g4?u;}d4gI#EwY7S_M?FojjF;rjQJp1>1unR^HC4;vOoZy%q-0h=}JWX2K;%D64!6;IKC z80^cnDpxMvaad9u#BUug-mY8djr#og65)v&f z>#8?j7>MF0yKd@U9?-VMvcw)79Ow_tZ3jF&H*`(Y6eRE+92y@_gxwob9#eIEVW`I;a4txV?|iy~ zz^ib|=&wSPioDsEd(?8w;%wg0vawSIUSKO}$z@TN%6VnFt{`KnTLjNadxo)Cd6tv% zo|CPxw3wfaBpv9RlW25go-7f|itiXx|TX$tJn+zy6h>485> z5%xK;(;tIgknQK~N+upATw(Uz>nP4AKhWcA(Yd~110BFPTj8|iZ#@5ssxV&%3~e#i z=9RvlAs5#6dQl`#xlR`OKn+yG9`^U$nACby`_kq zW{u`Wwm4S{YlbOoC5KsqnU@#lwMUr@(m-tOg4HVf(DnTE@+dt~4Fwb&v#ya#2v1BJ z9~;*_uKIT8$9>ulloRtqx8&}xyLxX~Ltik56LQY!T4_jf`Cm!mXLYc#p;_wWH@s)v zkvb&hV!yKKudO$1T9&Sym-Du6x>4*do7B(|vTt~Md)toy{dI`@P)nks90G}~+`T#l z>mShOb0JCTiT{P~G^g;F>s8^`h-Zu+9UJos%??zO|7#S5M%UKWDSYUf45!P@iL!Ya z#(30W!fxJUbeR6Z>P~!kAGeH_mJ&N>VWuemXT=+wAKrxR*l2lm(Zt`Cm1EXI$6P~Q zVm4n97EHcygEeQ3Ra;ZEsO68#$5@?5V@a{UQtw)_@zxti4Jpe1bthbnE_P(y_$=nm zE<*K_NeQPw==5LJ*2l$Mv0rm62zcBJ%HJOW#*B}Tqf5y1i@C&>^yTlJ2XluT15f6cj*^hpt45rp3}c zPAqCHspist8(XE}QeiN>^o5lS_p9L?ZA(i_X=!QjDb%(-gD`MYdh3tDk9l&n1FQ3V zSOJA0B`tY1aESws$!DtqtYvUeQ_oQRWJB*>KPyZ0vvt=~jw|FqhEo-X$&Mzs0I;ep zJBTwI zduHW6^!br=yf9_Ul}ht4o16ElQiiUosHmrlR3J?M8Lp(`*O2j)ti6FQtNma`B5^dT zZF;sq-QrGR4IYI>?=pMci=9({ocf$})soOlI5>u!HnHTQek1f?xC4xf{CcZ7C$2TQ zk~54T$4Zg0__)5m`U022VgbhnbDl^BCIQuuRQWOA?uUC=~W-R`dc zS=EHXV$6j9zA3K1vz#tMK|S#6SJvyw@|DfS&CR~^&0b;DPhLcm?z7TXP-&Ff(-Zo;J&dX^O?W_) zQn|Y|;0j?eQ{rb@MAQR3sHf8M$`-%VxtFkg)uG!2`m$8`AFeE(9dmXA4-XFo1%>#8 z8_@hBINX1Kp?%z4dR9^X-gskE)A81nOZMU0;pqDYN7Mt&`{=?4lD|vdCK%%Pj^&S% zZ#%F)>~0?LN7Exuf;6O~U4Qn<<~a5Z)#}clzd#%DDJPAvN2vFnhTc8qRgt%P{{7A4 zcV(re#*MCp$NX)<78lKm>gqf|$f$KVC>?_61m%t^Z?+HkPu1-35Nbon>+xJ!i2b7V zT#xU0tka=YH8pN)E{ zqcHBC=Wo*lpK0A!GTlRt3&nSr0)z5YlDJK(IWr3`{D}^i+S;0`vL1-j$UOe~qEN>a zru?C+>yFbKG&-M0ElteQXn=i+oAUt?<++$X$SM|$Yzo_p>?*W@tOijzA9KrlkB5L} z_MZ|w=xp%u|G9m{w{}Yuvnxp6$J#=A==Gy%mz=FjJdRSKI}{<33+n zm&(tRnDh7-J^-14OVEHlkmN|P0oV+D{~Qzi&3Un=@C{NRfSTHXk zzE-THRmGA970Xfn8r@D#WO-we?=CKsfcPCIJ_%s>4UE!_>9U#JKg+hB_Lq#kGA?8% zkY0y-2bv55i$5kid~IYA8xsb{Z9|h15^!g$5Cr~ovxVv@8oiAqOu%pxm$GV`o6jO= zpoB%AU0fT7q|<&owU5*1&Dc=tm#XlkW!TyBH)c1?ndpS^8#V11W3WHw=H^C9^|=0I zA#7wFRqp@tb-VWIWW~Yf&rt+4lmn}ieaFqi13Ft>_!}|%b$@AE(9W^tXdJfBga4F- zSSxGFu_X{(PK} z&bOO`c<1<9?94W-yEguQHUq&gLi#hr>vD2(EYxY0f37?+ocG==J=0dodqMte?|ZEH z&fb7K%7|}2M}^%4)x#4>zDq0QyL@;z?xl2OA}8}hF0ReBwJzA_kpd09moFcF`B;&p zBrmzJ@Ka}UXjV#uB9)pp!E6m*reJ9 z{17vsyO%}&G}2uX?}?8az;0P@!+8_&Colr)lcXU4V<0hB4i0=AFZpKEu=H^D}Je z9?vyjKlWS733JYTlAyev^sK|N+4hu-I(d#K3!)&rU!h}Sg6ANA7EK7;%@Ec8VNJ#h zp6)ba@^z^m5GO9ajz{dq&OW$HzY^w5OpBL1w*&?5ygEjKlsMqHd5zEOU@1MI<>fep z3GcbTdlRQ`tyY{4`%@Dat_gC|Vg9tTm%Hhg+o$~p;Hld0+TEr(TK4xqotlOGT}atK zyXrYlG;$x6)6vo>V4nOW?LYT)I$tg0;L7tdhfy_OBUcX(YWtoG;ndw%!wZy>mZ-{i zrm7S~__%}WpUzx$HUjhmMwm1g9q{dDOImwzqr!R4ZDN^0T|uEUQ^KcN*?borG@bqU z&X|y$5fS0$n?(sJcW?d_i3I^`{L?}2lGy^V8W%*yKGiX=No9q6H-Li#ur1y>w$4c}YAbS|Fw-E1flLYd>+KKxcsl1in#w4+ z$jHbp9sf0jfDul~=(iNbI#!ULeV`u&JS(WxH!p9=U5wgo;(<_sgin(&RKB#R1l|ES zzU^O*hS=j>)1jZC$^SVEfVGCX@BNnM8B9(Qa_iY0BYzdN=W{e+DzROsyjx;kTvKxd z9tH(~P4(=1cN~YI@&(>eO;gjozr@7EcA5xy2_N)TT<7pTOO}_+rQfu7UxDhzBZQ$$ zoSVBUm5xoNIs=T9(!hYNX}fa9H4-B8aloP>@LfqvS2Cuj0HXEojk{@gMMtrD9)L#R z+q61+QKyAADGe{M%bzss_52}ceS}U&CP&tm%Clt_^JO3@U7&TH1YSJoW`8f}8 z7*h<2gUJgnxVj&HuTu($T!8Mgbx`EJ4~jwUlN1`OPgXWSSPqxs01<2t@~_ih>gka$6BoVz3gMN7 zvwsfZndNL~fuk{5E%=U^0zPu2p0m-e%L%9Lw#Qxpgz;4SMBviCKwFvtFs3yN4%v|&?J}9r_;>1xtJ}5VIeym0M@%R2n28)r!)+Vx;io}E_O_PnkUW8 zw(#=udhXW}ViEs13t~NIig#G42(HiOp?-8-)?j8jNJ}q^FC?(OC{wiRbwKyW;m$~h z8F-6577ZXx6elcrOKE6selY38gP^T}`d&BZ8BwdBo88pZR8zYBTf?6ZbqouVQKa#R zy+vdFJl&)>CdHvwmKD^pd|W%_lVAQAW*BV(1JV;d=S$l-KR-{T{%G`{OAlx-$50r8 zj7--EV#ahS;4xVQuf0WwO=>yf-DF2cRzqQI2ArpZa75U&)kB(xWq?z1t}=UnQk2|v z|NDh7%!u8gBz9`&Hb+q9Q^DIYvQZ;|#vAIbK40v5l7*yTV_BW+Ny@g_U0bt(LLD4h z22-l1r>Bc_9$9Bc98FSKV{lcExVWHSi|H3WH*h}i+N$UTjKtWXbPd5ttSBPS1U$pd z$FH+ zM#%z?9!aPZxC_8GtD7HzgN-)-t#-j#>u!G}KDz66$Olfcbu98eBDiX7e4N|3vAV3* z9Tb2llFlUW?o*)y!l%qwi5YXhbbOKWW26vB|dwt zJ3Bj|hK<&S10vTJZD7ijWSrJ75d&P3vUS2V`9{g$=aG?-!o%y4X!M(s%1Bw#wn`7M zVzi6l(?(`uZTqH^bGwUV)%qMB&SU^pumZOWmNg>?dXET0~J7 z7ZvH28PU2~i;PZ`I)8iOPwI_GT)I$$b8-9_d-wwCYXTTTQxg_w##`ZZELwziufvsP zUkSou-0_n({Z?2uM`Kg(sdhLZqd*-zF{uWGcafOD;fc>VPAzwWfOtXA{j%!&?C3B) zcjJej!@CsDr3lxo7!+CPgJ!s2(t+4Y_3D_Gf${~Xd%!N?E|kFvq_?+s^(ZV6=)~Ha z&`&?0JxY=k5f6d2wwoJ8FCM<^x$yZdL+B z!(;-n;SxBbgJZ`L{t8tpBFHD|d$1^4Quu9}H8$lkSS(@uSre261q5crpz~6@5}%?e zG(fe%0yN5(zZMmJwwmF4$x1zqs`HMIHGm(*J!h%>xCn0BaK5!w266Xi#*cLY7;jcqvnn!}3K z(`Dy+SpHR0kFo^Jg+^v(hUVu-W(?aHSaWL;hI^W5oflWH5oOvdnjVxvr0U)7E<@v$ z5GGciDRqE6$R$Ik8c=PZ!Zb?VHJu($r^Xzgf$b+N`)AdU+#D zu%7b)1|*Q=Rs`gF&wz3cl2r@E+|G6vS0}n#GaRji&ZS^&?2YJ1kz2m+hQsXy)6_i- z44jM&mwIe8h%$hGW%pXY0;>plsKok+whLBo^ML!&)6mOFL+>YDjb>%-3UZaI<}))n z$!J?z&1!Es##u-Ru@3#lRrPaeGp0;Yd}(R%Tk>PBlsleH7dOOQ!6>4zj~=B9+@{db z+Pl?$s&#%iDdM&MSG z_G?Ndzo+rx9Z4y(ZOyZsp+6#3(J|e3I@s}DiOPAWx+epvcS{eM|K#dNktqqp4*<(%0z|KeQM+iMC_3SeYVUF~AO5M{4rLM3;` zl7-c}daZ(u7=u5kX{kz&tkBD}e)3N^yA!XeU8ZYJw3cH{d1G0v!*N6@>Q^s5Apia4$!$T;UWX`k`Vs zRj@sq!Oz&k_O0KNv4=B0H=K)O)(ojfU=9i_$boDMN;7a0tm3Kih+V)YR&|DOk`lx| zrLV-W;=spumow(Pc?r4Dc*DJA7Q^Y=!QK%=LU123;FNB`A9($r(b`&+xWveX*r7>t zJJthoc9_=2-e~0ONs3}Dp0hg;dv|%c#?8%~S?J%aG+NRqZ;4SKpq~lo9Z=>go%8YW zUZ6?}H!8W@v&dbg{ah#Dc5vj=ar5WScPn7y-{i7BR8_@(jps?d5V85+=2OO}{7y<%F^biXSQ|%zHKnx9JG+~N z=-zLy%?mr4j5fN2wFYt}au=Fr^X9F;gcVvqDn7Z*_YDhe4YNI~j+P0N8&{Zv3N^@M zYmmobXHA|Au`wU#ne|uL*r(J+a|V~5)8}`ob;z<~yKa!>p0I@W5lTQv9-VmYM!1S) ziZ@s&bcp)L;iEQdP497%)5|I_zz)SoTrNUaYLU>t3bxd1v+umC!dS++U{D zYWl9<;BIR1HXiSn)Hg8lJ6p)V9t&P~K6qWwT2Tm>? zh0O;M>2vOtb<@|}kIewitb;=!;s^f5Nt4B?DdZy1PSlpMv9pki7YmP+()crJfAPbj zX2xYc!|lLkp9&ep5dQ9~ZF1eJ<#*+*TGd%sAKb9?2>9Ust1l5T^!rliI6paF_+y`- zd_OR}5zva&LqB`H(8v$uxKq0EjF#V#4O6F$U!u0IQ_r4<7{6hB%5s516HCo>A^B~^KU zM@HtC5|}usmP-eLD7#)b8aVhtc@G591qAWdh+Xs>aQ&PVuB!RlMMdDJbWo9PC^}gX zxna$;iGC%*;~!r@IjHlU=QZ;tdjN9q?I48o!S(z1S6Epnc=9K{d)sr;o?3s9QB=GL zd=R!S_>GBPIS8-+7X>Ae%kP8RuEV!Mb+QgaNUy_J@5;y!hh-+frq*L$JcR5X9yRds zHnG+xh{C#T0$tE^8fyx zj1~AE&A;CZ3^id7N&ApLvGMEa;O?*evVOHE*2wh~Pkp%~;oYM8vi$fmP&1m|#2gV5 ztJF{-Jm%|%j~uQnYGgS2)qt}lr;MceJo=hu6Y}?2Sw8}wTK}~!*nJf|MYPs!HBJ23 zs;sS>86kj+92hGx;oYzNJY4?__;$TeIZsT|_l2!|i0ih*VA2s{<~DA1eOwPYiy0Lx z)>Cz;FBB+?28+24oD#Dxc+i45nKw7|unLNzHP(Q59AJ{lPNriwpz|-!C-s5QNSs>7 z?7D6iua}DFzKF4xD90^y9n_5Mp3MG&&I@8buPY&(fy*AEXg`QlN{~&d*TKC{`hCO@ zGqKEJvi!&|7Buc=SH}!i4sxO(vz{6g^B1~X z{E2phP%Dg$jbRilrPVZZ{w!vJIR5Qe}9S6P$TYEyxssYLb3 z&R03R1@2_+ehhQ3G1;prlO+ONw3!-A=vh}Uk1rjKNZZkD_GTvFK(+zA_M8*Q#1fV8Mf^XBAu21cbk_sxP}aoVG@V%jxt)#j3%(=-n?)5_m#jZZ{# z2Z){onV5t`7p9J>?L!ZspahG~LbM4bvBHCiwth!Z#c803o_1YIhFi2QAFjeNC)%CN z$TZwlt;3kImMdlYbFP@P23|xUNCat+2-=ro0i@+|wYXXL|K{NX36M>WXE){b11=jW zEr+9jucUuoj)VBGFGZQAHXIb|`EQ5a=IJ%|zRe7ZuaNF5EHUR>tj?bL9oo|j&fBLR zj<(%z8}7O>WKF*tsoB(P7e(n^y>`t^kgP3!y+$hC7vG+QHT-;MOVOiuCz!`o-QM=6 zTGlMVrY}oqS(sTAqydO=dqX#Z=v~yn^|q8nV}-TO{q3Lj!=kA2DgpY#D-0?VB6PzWRu@hkJM*x!fs^tma-nf3D5Bl0oc+$l5 zXmSZrKCs!?XC3L$zRG79MV8lx7GQfk{bIW>qhVjRI&T_W)sMJM_I%Xotf$FYJVt@B zlN{-#G5dGA-2=7#5O{i{uM=xkcOYJ>p8mpV+e7#6Z0S2z5byQ0!kL5Dt?Yzyf%&xg zMt*j;E@I=Tl2B9HKx!Okel#pH_L_(V9}#VBy{C=Z8P<=f&hX8?&l@*kfEIjZxHmPfvhy-s-W!w%y)G@s=SHg=#$r?PYdWW z!}&1X^|!{A3k07<#A?>K&9S@$v~$0xz0L&L?Co1#M|`j#3x#R^gtBVuD+^D?^`HWf znRCt9?DKsJ^XnK)7H-A~F-f#|1XA2qT|OPn9HyGe>j2USltSn0-ehU9NRM5t5`H+7 zH=lws7Mu*)xAZQJKa1iZX!}1V)?CCFmbs^@aI@2zP3A6gg?`-W1TTXuowUruA@Izz z0m7N7i@ME1{AWXV>VmwkL1Rn}n;X(JiLlt2<@p~o0mdh-*Sv6E|HXAMkPnd1#H8B( zv3Q4Ei{4>VdVE5HT0@!DubV8U9v*cd!=_OKk9b$XfW$?84OBM(A{FSIs1*mDn*ao4 z9KMdbEM3ehSc}*{sQ!62Mf36ajQI%f4rp^WCp15C-dDWMsn0k_Sd+HjDw~YLHt+d@M>#+bGmyuQZ{ zirEgtXAD$*qFQpai*skK#E>wq=2+^M#_7@_50hha!S_9*{qsW% zS;c9Dj>K@Obr4S-Kl|<<8i+GJX6*3l;d6qp$Bsm`%~V=cRCV_;aD|v~4TFKN2OhW2 z631PrKZg<@kTAi|I5jwN1_ri)QZGp{;d&Eit=fc9ARluyh(nk{2ka0cL=<;R3)Tx} zaQhJ*%_Mja;Y?zbUG-!#Ns_RhMA+zQN^IL0e)K=D_A2E0m4^LMR=new{%%N4bana& zAQqs_ z9c)Hpxjz5hlhMy+G{uJ7y-sK_DSGS{TrqK8Sh|52N5H6|bTYtM54uRRJ z;8O=x%2Rnk*18!oskXIiU%jRzFjRb*7Z~=6nSl1wD?OP|V4n4D+_tt{?YHE9KI;4B zyRQM_;l#wKKh!vVqJOCh7|5$hcud$(G1(wJd|{e=+;I#$K@t|aT@ZLT^t!tQZnk^X zw1-?aC-DKZsfPx*iqh2!^nA)f+7I(++7!G^rKWonJCd8iv)kB_MaASZW1FwOoRPA7MSxtZrC3;Qqz0RWqmuOr0^RCOrDw4PZ zQ@<^htbr8kwCQ)+55$Ja+Pv+rs`)R#WM-K&KZr$|dXI%qMLC9J5^V!CK_>^Zkc(Ye zwohBAN{ERH!zifmTSJ_bFP_Igf<6ITX6QMne+$o4IA(}ZtN(1CPtcalbIhl zK?bO+bE@Sr+t2N6C5<4lvcy|neA1=*`ss_nzw11oY?2!YCO6qbkT6P+Je10O=dZo? zj(7pIS|AWu$_W6EzlNHKgdl+EP~J81L1$zNGs8Ac7JWd&=XX+WziVpv0@v4M_Rc7r z3aC?6u+7LJc{YI1!CGek5;7rNL&R}#^!8sm6<40j6tHMkO|;+z|M_l;A1DK2{?YFe zIZ4$n8kNVoi)NTXxAlf45}!cBz?cifBIlg}tiMVi0vNabk@NcqmJ#0KJY8ODPe(NjN#g@8x$O%ulkr$vzq&9dqPN%J&e2`vC~=o@B_(4u30(6q`HU0J4jtx|;y#ZRC9d890;%yebthMJNG0{pJ&cL_na@UuYc` z{LPO29n9~XA_3o8_weOjHPq_d|63N0i}qYG@!iEgqw0JHPb&NUwd8dl55;p(<>6;| z(rc+E^j}%#+8*AW@V6cV^pwf>?Jppzi1I2G;?M(?bOuV*uTL4xMDr<^Np3N19S9kY z7PnQdh~voKD(fZ02SM5o?^qjQx-io{pGbc@mDa=uK-zgPPbA(rnBXww#Z4cP|C%2% za+(|a#3bs)*eQ;faRzG4>Bej>JI^I*frylkDzc>GXXkmrB)d&<4K;m_aowuyY$F{P zb<=O=2Nz--GTY4MQSbFX6k=g^2?1&${grhyU&jAZnErC?>V-5|wU|TXcll_F$UsMp z5d^!d3(T^U#~CgNTA`u4giclC+Xc&p`~K262;pz>I2gGoYRRK@e|6GY4A6H*eK;D( zC{Z$_d?fF9+5_sxfeef0B*gQdp)vLAHT<1T)4QfwkXNBW0WN3>B8miw`2I=;Nz=EH zPh<<_yz5@u)S!AC5HLRms-mSdPyKcvc9wm;!uc~YC&aMaLq%km@~&T`uqX1|1hNvV zi$7xOx8L%OxnHW&DF#d*M%Hyn;nzsg;%N_acwrNZ{seA#f?g3 z4Fs)7XKzvSw)KuCM?H5JguHMt9W7*O$8Jh6T{ni^=P)$$AEq2(9EIg20CxTS1n{}(+7b~|BPvZx`GFf|#k$*7rx_?&|5cgL8hh=6CF;s30 z5Ku>UR(}M9| zXuj3OSk{#hI{;wY#eSTlF^;EjNI#y{(6)-wbfpu-dpHOnANcuStceetsz=z*%` z(fbN1{eL=q0J%5Q%KRFlX7U*fC-1EQ{5N#)*gn1T8SdgsDITftDgBa1j5ua7>9c;s zhrs`w1(dr&I8qGX=!m%B_GPoz(h)nlm)d+M>1a|a9ket5WoDDcBU5Voht5Dz z_RSAHJ4pnm@jW~D31S4h+~d)oLPaV4Hs!;Nq9BjJNohsmNH@c4bNgdJ!W$2uac#1D zh*K7b^--?^VKJro{k@xf^ukoXTyG-1c0s!ExMn&;N4f&BJU5ZG?6cY7@i$QmaUqmM zVujPON98VX5zr~1Q@FiJof$dCPSPMeVRNa_lO3O;$^k(|@n!tE>+HTQK_TeXjq%R! z?){pt^BjyxW5JkIiE?JW#siOE1(a*q*XbZ(gsY5sGvalDktsi>r#T9@+JD6O#;C1i|iLG_lS0*6x#jeE&uZhX5*V3R-t-HSd#7b-9?J%46%_O$y7r|_l z*YVeMb_N*k5T`TQ-|s2VaDkGo2MGb%)FW$y$0M3((5!C6+O^fP7)y_vUx)$hY9vrF zLtR)vpl$C!R)w)ZXx0_rMn&N#hOfC^I_#pCuRNt>0i+akq>57ULp^@Ek!$8?${QJg zB#41_rPSJF4>k&+2WI5S>pT9LB7z zE`D=o4+aUrdTVCHA}mo%9kJ*F#-HIP3(piMGaj$b$?X8HcpD~w`A zECQB>!0rbz4A-|t&~fq^GzaJ5K;e@JoI z09Tcd>4*EHmUpXQ^$&-3xA78wOmMV#@j=>|br#e=;QK4LFZ3bA#UoaD3j$ehmSKen z)_CYBplz=Hb<04If36Yst;Y3$#v3CwTyY)d9WBHY~HCw>XAhqc$eC z9l+J|=Y4-Fs^4cP&a&Svc6?|nWto#I>Ad{llJ-zN8$K?-;1vA{_b2}>@#WBqPO!$v zr>FsYL~K)_#g9M`Riexz^7WW!{;D^IotpnT2EXfkf!ByCRgLH?kRGZJ@+ThX&DSjP z2WRc1F{VDfziJ4yz7v?j0_G63|7{4(g=soYP)mLX05?nY3^nKyC>tW=m=3gMt2HUs zPL-+A9e1H3<3%r+;Zr=E;@IK6Yi=e|fmr>K9Oi zpjEVseM%+{7N0aRXM#~+07L&j)Z%3;T^tSAtNz<-aUtt9_Ww&oVorUaEq*9NLol%x zM~N-r;U32f;Gn`;nfa6YN@=Xj=cfkF`cmDkHpB< zf`iU3aLQG@zSbIY;$pSt8q8*>GCLSg+#mpB^*`dfeF$>-6wjahkmP68)1TWy#UhDH zcDc^)5nAv>`9ia(Zd}y(+1n8;%fYqV_~l9@!t(g%zWpeK0(30< z(6y|DT(AJpJ^7YTXP=2}i1>riNI(7Qap^VV>XO$gF;RZ&hd{W#G&_pA1n|;&*9N|H zaGdne54F+=8bqjzO3X*V@UMLnCS%p0tSeZj$NIk^3Qqh?9Rdl44WfR}bJ+=cz79(M z0pWoEH-lO4C6C=SFpVG_kRy!fRLXg}68vzO+t18undrNOiW?hCOP^r5LhB3FGI_UG zm?d0K6JR!oV?xihahf$rY zp4{;=ikxJ&^7p)AMOqT~_o2v%Jz+=}V3dHxfz1F(13MH$3XS+NpYjc)=^tM)#!q%50 z*m!d`x+`}Re=H6QA&eEC3cs*=;HUgqVOFpcRWbPY+rGhcvn}%%Cv>vJ(JL?<1QEw@ z%G`)xZ)XIcw^)VPOR*$%Ili-g4s<+*t$gtOcby}yW+&-I2)}J=X2LL%Ll_f9T^Q;5 z#|k18UGgomU|p7jvGewgWPvj0UTw1&3NQxv$FdM`=MH}_m9C>#Q7J~T5}o)U4*zpN zmHB6@zx_RZ>Ei)qp%dnORsUs76l{GrE-mWfQPkpuL`j~a4bYtLz($bt^}YiG)j?m2 zv&LY^XgPE%n0aGDPoXiof|*`Xi0|P(4@&yUxKp!*k^ULE!jlvTCSmo85q@gk$HYRr z=PJ+gb*ku9Ru;`@nc8M5{@(`!iDyS^lL+v)%}8go5N%jh)ah^k7t5)&);n-}GPqq6-2QtK z+%8Lad_7V|@#<~xsz=~e4A_bfk!x9Ut$@G20VjB6EWILrm_dXFMR8S}k3le1!D_xf zF4uas)p=R007mr*G}pjTi;LG^9UM%@WL?-NUYAsdtBRb! z7aW#admoFY7r_>d4)^E%Zngn#dLImW&nGqu48=2dcV%Pk(7ww%+DA}S&zQy-Vt@LV zS1_!;QLjv0;qM1k!p6aG9{4OfY`Qo(ZDmCvs#_dan&+?24%9F}T#-F~bv%BnlW8pn z7ZsI&Vc~xpHY^KeW4XW{!NrZ$In(GDz`G;2RgRHWXGSVJG>;9}D>QJ$o10%n|FEed*abVM47sFwJn z_q?dAe`@3&8;AoRh{J31jz~4n1M8Wi9BpWYv8w@F z)@k^yRDYCu7mq1s#V%At%?kUgxjC=FitrJQ`gKP}Yq0$NJ@`x#6DWQ!?5}7$YaK`9 z;@7Oqpz@-@0R3EV@p6sSy0ObNyTe4k;t58)bN-nttPk8`Rd*AkgQNF>J0F9%Qznj1 zCt_E0UOF0!p9L%rqNqSBy0UWMTg1PTB@68Il-eoyx%kVhnd<1cqCz9w_*X53Q4pl} z%AgO~mvAyK?AP##}cz(I$0dVg)7QArM7TL|n_l}bFa zZu?q>C{IJ4A_!}Qzs=Bz;253NGC7F7B#6B|20(^ z;reA01iC^HNsq{%MyL3i1LTWr-pIOJuA#L5mF+7h6_Djq{jF zygcssmR1ym6cERKZbzSXrpB{z1W3ZhfNgFDeHtLMd`H8j-t^A=QfLLIM;M0?GVmCm%c`gqk>6g`8KxzfAF!Za0Nt1)@i#dgnMS4p2g^Q;*TS`!6qO~T}7lGj@V=>Qn zGRdh>K^9)=&i(r@O>Mk=pUb#22jZWXW$>;Y#@+VErv+!J?l3Yj6bdbriFyZ9U;H-{ zJk2}8MfX|-F=s|5VWEP|qNi;gCs|n#K)%WKItfc4_aAzgM%5p7a62sJd$3XS{tf5@ zSZg-YBI>9KM?3JYAN~W2JPsN*RPbGN-b3a$r~|pr%%ot_dzG!}?O(r_xAz4yuBJwP zu|GgINM|96K<~`c9lNf4Ks`K7HyCpUPN)t?7ZsO#{;616VaXsSASjW}@C*b zAPPuQqQnssB1sG&S&5Q!4kJNP$)JEBNfA&a=Qx5Wl5@^UavJi$%-r7U`QG<__5Hb3 z_x`w5uT@76!_1zwdv*8o^t0CP)ott1I-OOz5+uOSywS9y4l(}mR+W5aqmotdfm4jg z753}s2G9XVd85@63~3~|yU!t+=Lt6^s7_Dw=@nTwg12vJo@B^S~43WeVBLD`DGZ zDl+vZ-&ok!#SNA%v!PE9{l>fLMPB z3m8Xvv%(Sp_XMeZCQK<1bWsaGL2;nLe3BEKvk`Eb8TcF9c{=aV?DCt;xhxDn0|^?F zZ16^Xc;g{>Biny}BNvmn;Mg;0fA!U0Vfnv*;$?e#`y{RYHOh*g5MBmRx^AZM%rEtl z=Od!mGFJs`MDvR_B)zI{SZ_G#j9J@h*MiZNo)urE-RdZVLBD`ObKr^;T=*h;W^Mp{ z427}No!IJdhu;-%tnOTW@;Wl|{5nlec{fjUQM|Zr*`mk%EaW*zXIl)S96r`QU>UJ< zv87j0C~tLlc3 zRG5J#D|!m52J?z}#fdSX5Yi54C0?3CjGp=ifR2z?Z!A&*Q=B=5&vWYm?DuEU)e)O7 z+lVd}A zdDA?wZQpkVx#D@aqUu^+0ic6H@Wr{Ko?w=yoV_w?6|TuCqT+L_DB(rX3N+Vrpu{Jr$7 zcEc)ui+99%w_PX*qg1>AeCa}w)Ps_#Pmp;ojHxd(y4xenYN%3p@b|Y#!#v;#R;J-qNcqSP> z&u+qF{_QUlvEKvjLXz{s?iS1+|8@^>5ocf_KbA8TXVM)1@R4W56UA_s(y)!>LS&Dd zrX17kEBMnOBLY=buiTQt<&+}G?vSF6l{nq3UL1ka3gMhE;@6Hp5r0iZs~uy6f%NDS zqisI@ixLHWXZPM0@k6J?aWT!hS!Gz5emr;rz-1kFL?B;vAHE8wtu2L1x}h#bd|17{ z+9Jlq&0MiJ%oDHo#={pVZGPMn`;q!k%S?H~_?uwHK82QpT=&7x)3(SGcg@4M{E#tC zbr*cI{yaB>gTKwns@~Nw_!UgJbcCHfgaS&=W*|oCcU!K(JW`}|790=bx?MM!(PNqS zP5i;xrI-liRU06Ckl_EK$5kfmYE&VnV4~}>*n>C2BAhC^cFxJ#ZAv>+q%+rH-8u>L z%^Sa4i~Er84m=oz+kQSdJSHpuGc~GFfSc~1bH*$6jLAI|@OE+#5vLc!IjS zSQo7M>h-Wl>aHxh8~`%(vj9g(a_Zbj@7kM#(wD+Fg#i~}CmsbugE^g~^FkAm&}u-$ z5k)vqS8z(+j2!{(w79z;xcCUFV+p?QWD5Z@-!z z(0ny%A*65|7lQsQ!Nw@RvyP2~E#>hXP@k!Z%v`+IYWq<^k3dx ztva}x0$k28YQj`-Eld_2W`kKvZCNO@Gz|`U37}^=VyO^8b!bUwC0tu3C=gi73lJ$N zG;|x@L!MYD^PC0H_%#|G{maCB-ne-_?d$e6PgH-03M&sX-wMPpjmf-6k+k`@U}p3q zEb^%@%K&cK@wU?pp1LgvF!Ua@1vW!hzIiB8kun)jWW9AF0m|=!w`<3U7gt?kt|FX6 z2WMh5^!CR{A=9{f3)Y^m)_r{SZ#}(f;D#P3%gWycG`bEO7X5g$97k$zdcB`{A281H zYj0KvYR7G;(xKjXKqKIgIkio2k6ea3kd&uX+kJg%i)EvZvAlNy$>;p|ywWV9-W%QZ zF)B`}MlFoTMo*6b!l-Ei*a19E-8{Yl?1XwKuiRiE*bp%n$`iHOIIjCadf}Ri9_s9z z%u^mQSt@hi>qp~yIsMNMPQ)2lFCoQ;V|I;GhR$h4mU-i$O{is*is#`PD^R&`Vz7I1 z1iME9m=ik4-8$k-97jgeiJSKBx@MBQyboqDP)+BxdrX2%CfzgQ3B~2hjSyNLVJIhC z7F|tOy3nx$J#hyi+p|NqziLn$DDS0uIHb(%>r_JAI<^G}Wy0X-2hZhopKnxI%`S0^ zaHZ0>Yr!%(msV-vLQim$$b~@tibucfGXYvc9ZsZ?W?xz^g%pC6eS+(#zeO>4GbE0} zc?-M5G@+Njoe;piP88| zoDCJ(wW!l@q+UQYKhVNvrxGZO_wp)MHS?~(_d~CXYHnBAZ&*!d597Y-dOnqcE;e*4 z!fl8)8$c>P<~7sI--@!sUH&y@t>m_20;*=Dx=dYZF@fiLlK{iJ$4 zaXX3tb?mJuiNA%l5vbN6|@HG$< zoH{2lYrzf)<%_twEGZP|fv0MATg4%3#b3PKML1XxI^md;~BwypVsKEy?%@VlnFKn-Yc!lnZLT`>(pbK5EEn2I8aotiuzry?k zMRd-1NOZiJ!5>QH%G)~`Oc-3#xAcf2-6F^2^zHfTA-A}2NGK{ZuyJ{zAl9dq*qKEf zK#i!c?GkNy2~|G96QD=!u7~e@H~OHDvd%((k=NkfX{$o35bL3po>nai08q-HW`loPe*C_uMDPuE*roMEc`(i1h47jY2QKkW7D>D8A+nJuHjqGhj~9{x`;8_FzX+g!XTTxsL5zWP+(C}A@mc=EPCJ0~opNMf z+K_+D=0z86+J8B4`4|6wOl#kbPDnQ`cm0y$Ph@&xAr2o znosFCwAGrR*%;kT!XdNG+!5NJWaP?6j-#wtz@m#T7)1~#A94sY%}OA;qCKSdAZ3YQ z&RTZbsF2Tt*!vF+PCyFh2Atxn?U+<-og_vS=_CwJqRuP++h^WBtX-XKMWujpvh7Hv zM_5)njSDl21JM{N4)507@<@ZTw8OX(X@PTMJH(k$%o)tm7(~%EDbJ(8mDd@8TUsFT zyH~#k_EDk03*cJ)A(9d4_geQ|j<=BLPeG!OoGQ@01niO1s)FBXifU__wldg(35ggiG|D&U>GxZZ7z zpd)_15POYSdhv+8u69FGB%%JK17nGRv4r4qidglA?o2nafA#_#J^4Sq1zJZUDL+gI zT+9p-nsH>$Pf7@sLHMRYFrX?ZvKRBpZ(4z!CgFLZNq9W`1wEMJ-IR#kSAb+O7%e#f z0x6UylTe}_0ODMdgr*UPCaXDB>EO$=tgMcRhdVYmIi>o;p7AOi4%4&uju zzKG8gN*0>3MQ~n7x2phb!@wFDQ9%G>@M#aIN2v9~yx&aN?0be7@^N-NS9Mj) zV?HX2fWsYxI^=S~Ky?rZQ!&gVD^g0AKZy{JBmjM*m4I@&FulQBx3yhvW#vS zpIFNnWQ~4%{15=B3t^pnJQ^uBH0|W9;;)hYHSXSRxxia4L5D-`(F=QGL1+AKlyG}{ z>QDZ0f*6-BZFc&13Ab=|LTow&B$oIx2QgqElXvY_E-a~hQjxa>jti1i*h&Kw%dS9F zHtTD?Njq-{1^FY>aNsexV68&=RK~Al%K}yE|B=w1KK@^7kdv&N)vmu3vhD zOu)yF9}lVT4lR34Q{RG-tAY_V2g zJpkSs1K#R&RzZP*o_<^}B_^!-UuDJZw9(7JhsIbkt-PG>^=&OT)z>UR!TKS@dgcjk zd-mK~0IFC)3iaD4^$U(f?*oK}jxe}>klsB{upy%VUAPQP37^4~1)Be7s1r3x)Zk^! z_zb0oniLYQFcOnnU0oU_w?ja7X#+=I0Y~-*?Wc-ysnt{D(7eUp5}5o?t&2z+d9|QX zr+`k&q{M=%FeD_-M%Zuamm$RiSwtL-bdNIlq(UU>U)tJ3J zJbc&C5mq?DwTw1j}9xQd*epj=ZKCO}<4kRc( zYcMYN*<6Bk_rIk5CLg9z5bw2YI^-axbT7e*omiUVgH~IQ^=kp)*NC9{A_@vhYhdS^ zC~CkfkMgA$tPrh&MyRv8XW%+)DuSqI8nB(4w831XEj8JtGkix~3Uk4=(D{#+;x<$H zhWq!jj9qHiBiv@Wd+0!qsZ61X#ij4k!Xcg@-B|4eAFZ&M?>$4K|B(1RB5$)iZY053$(D_CPO& zu#DQfWF%L*^v*m&9cY$a%c`&jtplTqYb8RnMF@HkzpLbi?#G)I@YwD3$RXk5hd^a# z0}bDlhoRExW(<{H@n0gr2SmaTFgf*<EAR?ZK3o7!mA2I!<@!Vw zSrgOaoVg{omvW?0@Q5;2oomOOtd$zn-47D8mkERt(R4)rZ`Cq7cZ)9bqqf$5WN|?C zF;EFau7y>R?_xU`mjx5hphV2-_Cl3d4%F$fX_-C+Ambqiie+G4Ny|_z<6*nn8?y{y zuA?n_!6Qq|h%W&rC%JSKnw#itP7~#4yipjeW^4P9OG;w!BexBGG#u@LTzlA2W)M35|r^&vCNxh2O zT1MIbYMksru2h6sV_Lh#p9U=n2S#?3>mM4{kKlDcMXq>ug}*6AglEUhaw=3rq~(j= z@`prwPUUyon?;y7K^i(Km+WE|&m%E2C9)0^-qfnb~>IkG+HLCT}<9)w?^B z_nseRmU4lPV#AWGwLgOCA_P|jE?np@b(}B4;4OoG%>Inb(2}gj1-yFoYuF4Ejns71gdAC8IrjH}I$n$5fWJOae;CRT8 z!sMw*Z}!6Z4fmR#f&92sJ`oIEm)wh34+k?#!ze}x_+5pYtafB@=Emb{zu@XH6k#<4 zS~U+8xI54O3?pBm!#j<$M@tGUwH8J#B3lWcTb}+Q=X!|hG zPb6a&w@dcm7)QD?dEs%X-IGSyqZV)1PZN-%RS(5KGx+Edc3a_ntrcQ;z)d+zLU9m}}0@vbRr8dpO@8e_!o zvEx1FTK00=V3eFok8oNqImAwEVSZ*a}9=b z9nB>;so%=%-w}u#5BvHWvs=b}FSTEcUeMCFKawxbn(i1F%6O?sjOVoXPdItwCCqfZ z28Q4>Z{ZE|{OMjw8aPgO(=zfT!L(zv%>IKM_*PU}z&jE1?oRG|HI)Y!;p?;t zt%jfzxI|L4{zKRW=3DA`G7ZoeG_KeXNv4!FJ^vu_-Z-e^wlvu00msCfZ{*R*V~*NKzV zynCQJDShWw=&nkf;Egxh@#)r41Wq36eJ5|-e0oG-Eil~W^y$-Y7Sk<%G47nDxDE-X zV^D2pv2ny zM98V}OkUj}G`T-q{bjwVk(4)Rf7r>RW;&k(0xk3Dz$tY(N=_*3z+pD6AFo_*rAodg zxZf~MA@yyIQ0#&#*n5|X@fPNjnG`4GzA!ksB;TkZGBc8ml!@m@Al*H2)Ar~~2Bh^G zJ)NDWPQ7ptnJTg+n>h)?-~%5{vwqT>YotJi_9vI-^?7lfP2QQ5x)x7<%)zlq=y)R& z=Y#vv9S4wOID8lVy&q*Y^0IQO&s>C<_}O@aj^?VNP` zazDWFIXz25Z{J*~=^p%1sKTXW)PBMu68gQN$sE6XLqw~*75r}H`qWT%DWU3lO?3N- zvp$1*x5I-U(&qxpZajZ7r`uTKB?(<5-1DEc+Ima$Ke!u}Iaj~bq1KSVGe6is;(dGjJ3 zWjlWTURhuf`Umw35)BTzeQw&>ZNI(}@I&vE$i{ucLz}pLKZjb}VB1HoxZhAcd-?2y zF2^B#S$S5&)903hOqG`Fiu*5}SKE9NaxzpW`~<7BeqLRmmI23+!eP?9x%{T#NApeKjerVn&J4|j z&6qKmI^s@3na+)OywR7*wEFT0qa+UOz2891ke{ieln zpZuMn<=it%wWP6E2JU+88a<7N>2}k|{zXH2PC6{ShGCCzUoeu1^j_m=f>3WPb*E?f zR*Olqa0RhNcCT$Sg4yea@(a?RVnZlNUM2iU7T3#z=U|<6c65t6qH#`Xv-aSYrEBe5 zA%tlUQ^~locZbj_T$ zBnm>AwE3?_zX%OC{ayK_7pS<}O%>J=NDbxGid?CRZ%<0)Ji0*F$oox4RnWLG`k{(O z-p!{r#rZ?Y@)17kC<$5l*%v2#1uBGN$WI?Q2rO-~Eb$%3CWmjPHTmW;dYR?0D9=mO z^Temk-TM+HPx^$%OJ?)d>MOPSrf_0CkyZbv`>7kbCsMVm@&%m?M`4TCnwc?Oe*OFi zhWK;SE%ed8rgl1#)m{0Ckw6Gd(#47v(zBW^Vw+3hmlj(@ola^$j=OX;G#pO4ajDAB zz;)qeTr9(G&ligGk_KFrwyPD{(1lBFuUu} z#~prp@%Jzq2nqP^QHz_7JbVy9rJ&3YLagHi}4I z1X4wv+9Eo9YKfKNq)Q^%n;n##Rw=+GKYcMHv~rTgv6!xtgf**I2|0W6KgRz*@JulN zmCglAsp7n`0v$?6rCQD4d|}$tr-K+p%N=I2Kb?NlaUt{FL$&zkUEAD^MaNjNMYA+_ zd}~!iMn;ZY@kUJF0R{fmxc#hRN6kovoZHHlE4A+5&o_mqx?nFSst#7~umaODF`kzl zMw*|O%8AM3r6*-Z$i--?maP0J>9}w~vZSfBDVA9i9|4!F!_YsxdpFO;v@4%ZhF`Ed z%V8TV_e+gUHzJ6=CxswzhFFTk)F3!~;3g2$+_Jd;brS1N+ z3<<7JHdK5Ylrqxpdmp``axrb6VUK?lpOzYIe}vKqi_LdpySfyI+S%N>$9+MHO^Hfo zCxTPYLne9$U;7|kvR_0xdouhjdaVQ zrOR=s^1O*U$GX^}EPAn|WjR!7deYJ$3*lm7^5kb*YLDxNi7j6hWCUCFSRWe)a1A}a z^C8d-<1=Hz&(@Vl_eFjK6@StT2de+JQ!pB8D3{CHM!UX;*+kg z;o*@8{owOy3)D|qLpx@cJM($Za@{`|}{)!3^rN@OutB(moMkmp7n)1?z(l$7K=a@+Hb&Qk0#c9($Q#w!}SEy_N^ZWpfHAD z0Oo%_xrka)!=`prw*8dwj|PkW!z6JqV}ITvr`aF;gV6=Lyk%V~HdtYuO&cn~&o;#h z_U=1ILTmVEJWm&T2V890Ud@eie2@9AP?1h@3T_Hn24QXV3AUBpDn6P#;$&q#Q6+#j zkZmo+7_Z)qnc{fiNcjrtQd`sNrKHR1ym6uqwY^HX3r?CZzn~yAI zqCa5m#ej|Wtf7*MbO9l!sAdLif`#Xj_(!^T$rL2B|229D?W4LwrJKxl zIw#8%7G%Q@3jKU^dMVP}I97Ilt35=tH0(0>>w{^2WNV>*Z3rTGU2hit)>$k{&{P%A z@m)JXZ5&G9xv%xZKkpu_W|$@uSSd6b>5Cm6UZ@Bnx7ON>8%;29R@oe#OMu<=_S~%$&!L<2 z%f%Knc#ZOqUTHqlNs^lxQgrHt4@?BPt~Qb^ASc8Rm(RkioqzpCg;9r@QMcFOi_vnD z@9H!wfBD$2Bh9P>-AkEi3(1tjU&9shvktJ%!`=2d{%_e19~CUMv3r(im&yHIpTeOF ztn@qEb!0l;MValgIXZF0lQZR(9Wbj%{BVuq)HUxb<>Ir+ktDDz<7N`L0_(i8(e z*@M`hkH0DSAB*kZ9N>{O>!%`d{hmk`$_FlHcRuos3NH^Pb%L-Lx2q?qw-L9GRRs)D zszR>z|D2Ynvm{7gk##qm5TeQ;n1T}%`*+O7Dud7RHHF9&d4W$s?15DPXsG3)M(mmv z!oEG9uF5;WDtFZEN*2Vu>zG0S?Sfg_{FC#ib}i*n)b0oW!KS_r<7lUv^q~?@lWRl6 z*U0s(r5@hDFPxohSUrP<>uI1rl-ehO|6%rm4uY>3p!&M*6ao4;^#n^Z=q=3XN;1q< zmb=KpsK9#8QYB80>dwV!j~mVp%!^w0ciq{-CHcY!eXSchU)5`6y&c+N+9%jiS8RNz z;bAjb!ICylo@|D~M2swDEHq-g+EBR*pW136_Gk1o9;=c(-Y5=91mLaXv#Ahs zH1WZ`Iv5gd#+PHi>@Oh~-dPgzRx;W%Rb8k<#-T;pLS<(CfrV80go1t@ z$zNVMpCjCd%HB(C`@ac@a*{crUF6zvZCQ`qCi&wIKo9r<3#zEnc(r}>{S|M)v>UR> z0QF#ie0bZYL^33DR8M^oA~mpiti-;YK_lsjPUH!PPOnURp^YesfK3ZffRtsli)l_%jHOW4llU$D{ zOmS7MXM;#y>Vx}E*rPp`!>3}+mAQBloNi1#4G+JUkx@|Q#As?N)7LLy-RIoH&m$P( zS8a6XNX0u!vmuO{v9Bf$F{RrJL(=zLXC{2zhN|6mn3|6GCM;V@ZLc@PLX>hx7YH+% ze*$csZn-2Ng~z5mQj5C$)=HpYsPEz~lrfrCO^-PJ8#q4fxVc0(_-L;tlqwT}FemzB}a!XsSvVyuW);@D!J? z=FF7Eoq0lm;F->LmY$<#t@iJf4yX!eytixlL5r@`?-Ee^a=4=*;h~ytp37{D`WN6Y z?V=@p<+EWyEt6Gia|cBNqK3h(Vb)UKIjN+l1uA9vPGto;60hcn%AbH2Ci)aTv}7;d*ac zx`lmya(1h~VPl;k^!pL%Vq9v~=5p4EjlENXt49R-sfWQGC^jk04qnQn>r70Cr>h;h zmIh_Odwf51N?3cM=3)fiSy&=G>`))NdTi~0!tE3_Z8}Dok+y62sr+<(>91zaRv}$l zkTN{X29{)9K?46V9TY2jeI>U%xR~4ScSxoWhnh>7_mYU6zk@ zLbH<6j4s5yLNIM(y*RQevHKcgF+o_ZcPOytu^brL1^<_z!}eHuWqow5{&H@o@gypR zgoJut{<`e_l^vj}RjM8;m~zv_1E?8~qgwgzakIt7ngELA`PSmWt}As39O}K5!`s88 zR+HHYKpTXcqP8&GXbOn0L;zEVjn^F*##7&xQXOEzfVp-og5@_j~r zRb?&&Vn~3_LU6>{)IzBM)XQsc5Vt#RVkb3L*IhI%y*)IVU@gC4%b_P69Z<_3^UL3* z>WE{V@8ZS}%aQ4%Zzyza#wvQ?zEW(MMu2xOY0+qK{?n!T@3wqVzM4;d6qM6<3KI6O z*C*1%At{Rvc1zHbHi%@20n**+X|vf_Z&)@Wu8O^2l0juBPeuwf$3Ik)Nt_HXY@2Vb zN)@wqt5_Xfm$VvMoh%JLABOU>!q;_oZ(rL0tbBo~0-o3NBeD)Kf2Mf_?Sp$Cv%+XoX|tytAgh<@jYH$ ze0%)6j1RcLuTi_WpW8={11%G)1C$sQ1)EB(*)0OBxvSq-nYq2S3s@&PVB%e+M}c_L zPAnt3ZItZ5ZGQ1CoKv>0TW-O|@aZmE3n#D^eh+_^qyJay;z&>eC)0&XwhkH#2`qiv ziWWnc374OahD5+^BDIrKA604T@R{8X9u=BLY_)Di+6_~QG%2=xXtDyBC~%}HW9|95 ze;f%OBj87hwW{?rNc6A2O%HF;CR*)`FKn}rO{@Yz(%YMe(kYc@Y^SfR0hhI#Y0$H<;c6l}(>rIt||0Hvfm=1MV(T8VW_;t88DRM9`BwSG;`NG6) ze4p+f3+L+EEcm1PpvzEvkU$FRWnhzDs@nbM8~2aQXlDhQi!*dqOJsf=G2zJiQJ_oL zX(yWLXnJQp3GuO^qysz-RY2-|KB>szTYTyU0Sx*&Qfj@nM|b&ZQ5W4^M)F_PCptMG z>^C^%p@Qp3^dtI9TLBNat*PDnX-}CS*j!`z7|(prgq*CSM!A22vr$Tai!!*6N=>KT zRc0)7m6p}1HdrPYNqprW>CT8-xLjRy8*K|}A*I!c{alD7PRzrc&kBwcOgd=_%$CFk zzUZE%zHR-c@sCv;g3*f0@c(z(J#q^~3I#o6!M}bGbZ@UkGf*T9)A!g3rP1}c-7?mD z3b*!N2uYN&u(zyd5c|DPmKkHUmg1+Y!KpQJ>=m2!&Uk{BYn+9rFCp2ex z`9U+T+O*-IE)>pynLV(h?Bx13fx}lzS*aoR>(Qa!g&pzSO)Rkh} z+J*pAckZ;Q_-=%1XBsE(#49rDL~8H$*9CQyX#CK7`Y4C1*npFzR$8o?QY})#_1LPw z-2M3h7=8CrKSbCLu%3*eg0o(_3}|?2py(qyiQm-RCwr&j;AAj!G7y_72dYKD`nN#f<-$=XXguvXrs-6O3;u(Tzpa3W=KwYO@JP7>M@; zF1G{>v8wb>ICB*CY~8CUatQ|?#fO~Dix~zdb^l!9u($EUH|KlWLq1QMX&kpe=Q${a zePU~A*Eku(Ew&+D6#j5z;t2GP$goDwGU!p!aM-)vJvo`@!O)W{hzz&EUzTq|;0g>^ z^$apxK3w3Hs3Eteg-5|>&`(F2H^q%s26SB&%xMJhJFgxfd9m`7`yGGR57l0X8al3rXA{lzrAM|3dBbC5{!%s%aU z@e`#}+7dX6!vyZN^niiP#`x}z z3LBMH(`k_gE0wfko``{oR);7h&{UxOP^!JTN8I!kx>W~KE&90RW#G+V%k(! zaW{{zW^t{EFTG(@%!!}ZEvzlG?+JIyn=jF(pJe6}t?SLVmkS=5GX=9!xo2H;IIVXvxf?P>XUGs^_;yii<|BV&wr9lWW*J0|c7S5b%NsL^jO;jfn`wasMO)LFkhye5k ze7Ps{@n)+&(2uBmuPaPWREB!KGJX^az8o4iT90?+ztAFSy?Y=d zv+sh2O*9dZ>;+Rws7>?LX&+{SDM1g1K5Dpk(B%=K$H#Mn$&*pmeeX4`4%B5p5{F)jSVxJtx5yHf`Yqe)!?2)t%V(T7ZM=z+tpF8#l z$cs#0uBBKsor#r&@=333ft*f?{DsdRT~Dq^+G$*IcatpN$nT7IRUAq8oN094N0mSh zqkoRd$8jzzX;enLIlNO-EwNi|2M&T6cCi6Eqox#l3Cfl$yL$%K6W%~w$j*+;`yVqd z+%C*Z;y;rFkg1%J_U@tR`OS2wP?3}pUBIhoAR%Jcy8>#-G@rTT?qWj#Z4GL4_{V{& z?xoSl3P()>#D@IN+8^zBeA)z=ZFQ4SX^shjuQj>Y(cr3DRT0aZ3w|%QRYbR$Q#_WS zoLSaU#RJ`}4X!x4q*Z1vx4;i+{fD0LSRYa#>H~4daQC#0gEdV`e}$ z+@R7Nfy&*N8TJr24gorDiK>xz)Na02^FJ3;_48DtNCKh)B|o+>yD@cO4HMb7EdliF z&VDA;w*WEVn=nHPsTrEt#>EqLV@1Grf!A@f*ns-0BE90vTX<!-uV3?;CCEXL#Ai!OZX2o;R>n!aJgOX1?0f^!9sLXh3?y(|69O@N7Sxr> zLnU{jI(c<9B^-LMrK9`X+#iDRn#QfqVf0;h`^jBHNliP%|4b$VKw@#~!W0(aKOFUv`BPnieii3o zD&W#)0d!}n#C8dU65Src)?Evcg2j-cwkg+v4YI=uaDy5wori6;8;Jkh;17{eAAV?j z;yln#Vc9S2UZ@!MkL334>>$Zu<-bx|!hcC=uyXWwt-v+n-_{D|zaIY2LJ`T8|65C# zU9~1Egx#NC6+CWVao7&go!Q`FV*ayiw>3Zi?L1NbCKrh(nQnXS$RQHH@5qXd$sv-R zZ&t{%>VK9+|L0}V|EwAR|FEFGM_e4tq1=D*C`lD-6@B){MXW%8~ nEnNn3_#tNR&&HA;XRI~0B{fSAJj;YvN#t)RU(dYy$mf3nB{zb0 literal 0 HcmV?d00001 diff --git a/docs/bucket/replication/README.md b/docs/bucket/replication/README.md index fef3dff20..a550b731b 100644 --- a/docs/bucket/replication/README.md +++ b/docs/bucket/replication/README.md @@ -88,6 +88,7 @@ The replication configuration can now be added to the source bucket by applying "Status": "Enabled", "Priority": 1, "DeleteMarkerReplication": { "Status": "Disabled" }, + "DeleteReplication": { "Status": "Disabled" }, "Filter" : { "And": { "Prefix": "Tax", @@ -112,18 +113,8 @@ The replication configuration can now be added to the source bucket by applying } ``` -``` -mc replicate add myminio/srcbucket/Tax --priority 1 --arn "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --remote-bucket "destbucket" -Replication configuration applied successfully to myminio/srcbucket. -``` - The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely. - -When an object is deleted from the source bucket, the replica will not be deleted as per S3 spec. - -![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/DELETE_bucket_replication.png) - When object locking is used in conjunction with replication, both source and destination buckets needs to have object locking enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption. Replication status can be seen in the metadata on the source and destination objects. On the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETE` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk crawl cycle. @@ -136,6 +127,24 @@ It is recommended that replication be run in a system with atleast two CPU's ava ![head](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/HEAD_bucket_replication.png) +## MinIO Extension +Delete marker replication is allowed in [AWS V1 Configuration](https://aws.amazon.com/blogs/storage/managing-delete-marker-replication-in-amazon-s3/) but not in V2 configuration. The MinIO implementation above is based on V2 configuration, however it has been extended to allow both DeleteMarker replication and replication of versioned deletes with the `DeleteMarkerReplication` and `DeleteReplication` fields in the replication configuration above. By default, this is set to `Disabled` unless the user specifies it while adding a replication rule. + +When an object is deleted from the source bucket, the corresponding replica version will be marked deleted if delete marker replication is enabled in the replication configuration.Replication of deletes that specify a version id (a.k.a hard deletes) can be enabled by setting the `DeleteReplication` status to enabled in the replication configuration. This is a MinIO specific extension that can be enabled using the `mc replicate add` or `mc replicate edit` command with the --replicate "delete" flag. + +Note that due to this extension behavior, AWS SDK's may not support the extension functionality pertaining to replicating versioned deletes. + +To add a replication rule allowing both delete marker replication, versioned delete replication or both specify the --replicate flag with comma separated values as in the example below. + +Additional permission of "s3:ReplicateDelete" action would need to be specified on the access key configured for the target cluster if Delete Marker replication or versioned delete replication is enabled. +``` +mc replicate add myminio/srcbucket/Tax --priority 1 --arn "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --remote-bucket "destbucket" --replicate "delete,delete-marker" +Replication configuration applied successfully to myminio/srcbucket. +``` +Note that both source and target instance need to be upgraded to latest release to take advantage of Delete marker replication. + +![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/DELETE_bucket_replication.png) + ## Explore Further - [MinIO Bucket Versioning Implementation](https://docs.minio.io/docs/minio-bucket-versioning-guide.html) - [MinIO Client Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide.html) diff --git a/go.mod b/go.mod index 6671cc471..d25100f80 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/miekg/dns v1.1.8 github.com/minio/cli v1.22.0 github.com/minio/highwayhash v1.0.0 - github.com/minio/minio-go/v7 v7.0.6-0.20200929220449-755b5633803a + github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac github.com/minio/selfupdate v0.3.1 github.com/minio/sha256-simd v0.1.1 github.com/minio/simdjson-go v0.1.5 diff --git a/go.sum b/go.sum index c065567d2..727e2114a 100644 --- a/go.sum +++ b/go.sum @@ -46,6 +46,7 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.29.11 h1:f1QJRPu30p0i1lzKhkSSaZFudFGCra2HKgdE442nN6c= github.com/aws/aws-sdk-go v1.29.11/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/bcicen/jstream v1.0.1 h1:BXY7Cu4rdmc0rhyTVyT3UkxAiX3bnLpKLas9btbH5ck= github.com/bcicen/jstream v1.0.1/go.mod h1:9ielPxqFry7Y4Tg3j4BfjPocfJ3TbsRtXOAYXYmRuAQ= @@ -235,6 +236,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jcmturner/gofork v0.0.0-20180107083740-2aebee971930/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -308,8 +310,8 @@ github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.6-0.20200929220449-755b5633803a h1:duFwQxzoPzBt4LbaftSVVTDwRrpuMIKI6XoJ5mKSaOs= -github.com/minio/minio-go/v7 v7.0.6-0.20200929220449-755b5633803a/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= +github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac h1:0meQIZTQR/JkAxfygReKcb15QINBKpFd4LII2PT5jSY= +github.com/minio/minio-go/v7 v7.0.6-0.20201013215222-14baba9e61ac/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk= github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs= github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= @@ -330,11 +332,9 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 h1:ULR/QWMgcgRiZLUjSSJMU+fW+RDMstRdmnDWj9Q+AsA= github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls= -github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -349,7 +349,6 @@ github.com/nats-io/nats-streaming-server v0.18.0 h1:+RDozeN9scwCm0Wc2fYlvGcP144h github.com/nats-io/nats-streaming-server v0.18.0/go.mod h1:Y9Aiif2oANuoKazQrs4wXtF3jqt6p97ODQg68lR5TnY= github.com/nats-io/nats.go v1.10.0 h1:L8qnKaofSfNFbXg0C5F71LdjPRnmQwSsA4ukmkt1TvY= github.com/nats-io/nats.go v1.10.0/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= -github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= @@ -375,7 +374,6 @@ github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4= github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -387,7 +385,6 @@ github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4 github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -396,7 +393,6 @@ github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= @@ -421,7 +417,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.1 h1:voD4ITNjPL5jjBfgR/r8fPIIBrliWrWHeiJApdr3r4w= github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= @@ -476,7 +471,6 @@ go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e h1:HZQLoe71Q24wVyDrGBRcVuogx32U+cPlcm/WoSLUI6c= go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= -go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -500,9 +494,7 @@ golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -514,7 +506,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -532,7 +523,6 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= @@ -545,7 +535,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -567,15 +556,12 @@ golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200915084602-288bc346aa39 h1:356XA7ITklAU2//sYkjFeco+dH1bCRD8XCJ9FIEsvo4= golang.org/x/sys v0.0.0-20200915084602-288bc346aa39/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= @@ -595,16 +581,13 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200425043458-8463f397d07c h1:iHhCR0b26amDCiiO+kBguKZom9aMF+NrFxh9zeKR/XU= golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f h1:7+Nz9MyPqt2qMCTvNiRy1G0zYfkB7UCa+ayT6uVvbyI= golang.org/x/tools v0.0.0-20200929223013-bf155c11ec6f/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -617,14 +600,12 @@ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoA google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8 h1:x913Lq/RebkvUmRSdQ8MNb0GZKn+SR1ESfoetcQSeak= google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.22.0 h1:J0UbZOIrCAl+fpTOf8YLs4dJo8L/owV4LYVtAXQoPkw= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= @@ -653,7 +634,6 @@ gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010= gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= diff --git a/pkg/bucket/replication/replication.go b/pkg/bucket/replication/replication.go index 47ac4d544..673826d94 100644 --- a/pkg/bucket/replication/replication.go +++ b/pkg/bucket/replication/replication.go @@ -46,6 +46,11 @@ func (s StatusType) String() string { return string(s) } +// Empty returns true if this status is not set +func (s StatusType) Empty() bool { + return string(s) == "" +} + var ( errReplicationTooManyRules = Errorf("Replication configuration allows a maximum of 1000 rules") errReplicationNoRule = Errorf("Replication configuration should have at least one rule") @@ -156,8 +161,11 @@ func (c Config) GetDestination() Destination { func (c Config) Replicate(obj ObjectOpts) bool { for _, rule := range c.FilterActionableRules(obj) { - - if obj.DeleteMarker { + // check MinIO extension for versioned deletes + if !obj.DeleteMarker && obj.VersionID != "" && rule.DeleteReplication.Status == Disabled { + return false + } + if obj.DeleteMarker && rule.DeleteMarkerReplication.Status == Disabled { // Indicates whether MinIO will remove a delete marker. By default, delete markers // are not replicated. return false @@ -165,9 +173,6 @@ func (c Config) Replicate(obj ObjectOpts) bool { if obj.SSEC { return false } - if obj.VersionID != "" && !obj.IsLatest { - return false - } if rule.Status == Disabled { continue } diff --git a/pkg/bucket/replication/rule.go b/pkg/bucket/replication/rule.go index f3d76ef1b..a96f53746 100644 --- a/pkg/bucket/replication/rule.go +++ b/pkg/bucket/replication/rule.go @@ -45,12 +45,50 @@ func (d DeleteMarkerReplication) Validate() error { if d.IsEmpty() { return errDeleteMarkerReplicationMissing } - if d.Status != Disabled { + if d.Status != Disabled && d.Status != Enabled { return errInvalidDeleteMarkerReplicationStatus } return nil } +// DeleteReplication - whether versioned deletes are replicated - this is a MinIO only +// extension. +type DeleteReplication struct { + Status Status `xml:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteReplication is not set +func (d DeleteReplication) IsEmpty() bool { + return len(d.Status) == 0 +} + +// Validate validates whether the status is disabled. +func (d DeleteReplication) Validate() error { + if d.IsEmpty() { + return errDeleteReplicationMissing + } + if d.Status != Disabled && d.Status != Enabled { + return errInvalidDeleteReplicationStatus + } + return nil +} + +// UnmarshalXML - decodes XML data. +func (d *DeleteReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) { + // Make subtype to avoid recursive UnmarshalXML(). + type deleteReplication DeleteReplication + drep := deleteReplication{} + + if err := dec.DecodeElement(&drep, &start); err != nil { + return err + } + if len(drep.Status) == 0 { + drep.Status = Disabled + } + d.Status = drep.Status + return nil +} + // Rule - a rule for replication configuration. type Rule struct { XMLName xml.Name `xml:"Rule" json:"Rule"` @@ -58,8 +96,10 @@ type Rule struct { Status Status `xml:"Status" json:"Status"` Priority int `xml:"Priority" json:"Priority"` DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication" json:"DeleteMarkerReplication"` - Destination Destination `xml:"Destination" json:"Destination"` - Filter Filter `xml:"Filter" json:"Filter"` + // MinIO extension to replicate versioned deletes + DeleteReplication DeleteReplication `xml:"DeleteReplication" json:"DeleteReplication"` + Destination Destination `xml:"Destination" json:"Destination"` + Filter Filter `xml:"Filter" json:"Filter"` } var ( @@ -70,6 +110,8 @@ var ( errPriorityMissing = Errorf("Priority must be specified") errInvalidDeleteMarkerReplicationStatus = Errorf("Delete marker replication is currently not supported") errDestinationSourceIdentical = Errorf("Destination bucket cannot be the same as the source bucket.") + errDeleteReplicationMissing = Errorf("Delete replication must be specified") + errInvalidDeleteReplicationStatus = Errorf("Delete replication is either enable|disable") ) // validateID - checks if ID is valid or not. @@ -146,6 +188,9 @@ func (r Rule) Validate(bucket string, sameTarget bool) error { if err := r.DeleteMarkerReplication.Validate(); err != nil { return err } + if err := r.DeleteReplication.Validate(); err != nil { + return err + } if r.Priority < 0 { return errPriorityMissing }