From 29eea52e1497524b1e79095c42fc40639405e7bf Mon Sep 17 00:00:00 2001 From: Krishnan Parthasarathi Date: Fri, 16 Jul 2021 09:38:27 -0700 Subject: [PATCH] Skip transitioning of object versions if inlined (#12705) --- cmd/erasure-object.go | 10 ++++++++-- cmd/storage-datatypes.go | 11 +++++++++++ cmd/xl-storage-format-v2.go | 26 ++++++++++++++++++-------- cmd/xl-storage.go | 4 ++-- 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 99b7f8d9c..c21b5e764 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -267,7 +267,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje // due to a bug in RenameData() the fi.Data is not niled leading to // GetObject thinking that fi.Data is valid while fi.Size has // changed already. - if _, ok := fi.Metadata[ReservedMetadataPrefixLower+"inline-data"]; ok { + if fi.InlineData() { shardFileSize := erasure.ShardFileSize(fi.Size) if shardFileSize >= 0 && shardFileSize >= smallFileThreshold { for i := range metaArr { @@ -816,7 +816,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st if len(inlineBuffers) > 0 { // Set an additional header when data is inlined. for index := range partsMetadata { - partsMetadata[index].Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true" + partsMetadata[index].SetInlineData() } } @@ -1393,6 +1393,12 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st return toObjectErr(err, bucket, object) } } + + // object content is small enough that it's inlined, skipping transition + if fi.InlineData() { + return nil + } + destObj, err := genTransitionObjName() if err != nil { return err diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index e212de928..440e0143f 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -181,6 +181,17 @@ type FileInfo struct { SuccessorModTime time.Time } +// InlineData returns true if object contents are inlined alongside its metadata. +func (fi FileInfo) InlineData() bool { + _, ok := fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] + return ok +} + +// SetInlineData marks object (version) as inline. +func (fi FileInfo) SetInlineData() { + fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true" +} + // VersionPurgeStatusKey denotes purge status in metadata const VersionPurgeStatusKey = "purgestatus" diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go index 0331f5cc3..0b34e4ec9 100644 --- a/cmd/xl-storage-format-v2.go +++ b/cmd/xl-storage-format-v2.go @@ -970,7 +970,7 @@ func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) // UsesDataDir returns true if this object version uses its data directory for // its contents and false otherwise. -func (j *xlMetaV2Object) UsesDataDir() bool { +func (j xlMetaV2Object) UsesDataDir() bool { // Skip if this version is not transitioned, i.e it uses its data directory. if !bytes.Equal(j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus], []byte(lifecycle.TransitionComplete)) { return true @@ -980,6 +980,19 @@ func (j *xlMetaV2Object) UsesDataDir() bool { return isRestoredObjectOnDisk(j.MetaUser) } +func (j *xlMetaV2Object) SetTransition(fi FileInfo) { + j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus) + j.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName) + j.MetaSys[ReservedMetadataPrefixLower+TransitionedVersionID] = []byte(fi.TransitionVersionID) + j.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier) +} + +func (j *xlMetaV2Object) RemoveRestoreHdrs() { + delete(j.MetaUser, xhttp.AmzRestore) + delete(j.MetaUser, xhttp.AmzRestoreExpiryDays) + delete(j.MetaUser, xhttp.AmzRestoreRequestDate) +} + func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) { versionID := "" var uv uuid.UUID @@ -1211,14 +1224,11 @@ func (z *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) { if version.ObjectV2.VersionID == uv { switch { case fi.ExpireRestored: - delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestore) - delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestoreExpiryDays) - delete(z.Versions[i].ObjectV2.MetaUser, xhttp.AmzRestoreRequestDate) + z.Versions[i].ObjectV2.RemoveRestoreHdrs() + case fi.TransitionStatus == lifecycle.TransitionComplete: - z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionStatus] = []byte(fi.TransitionStatus) - z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedObjectName] = []byte(fi.TransitionedObjName) - z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionedVersionID] = []byte(fi.TransitionVersionID) - z.Versions[i].ObjectV2.MetaSys[ReservedMetadataPrefixLower+TransitionTier] = []byte(fi.TransitionTier) + z.Versions[i].ObjectV2.SetTransition(fi) + default: z.Versions = append(z.Versions[:i], z.Versions[i+1:]...) // if uv has tiered content we add a diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 7cf3d41a3..6a97a255b 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -1099,8 +1099,8 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str if readData { if len(fi.Data) > 0 || fi.Size == 0 { if len(fi.Data) > 0 { - if _, ok := fi.Metadata[ReservedMetadataPrefixLower+"inline-data"]; !ok { - fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true" + if !fi.InlineData() { + fi.SetInlineData() } } return fi, nil