minio/cmd/xl-storage-format-utils.go
Harshavardhana 7bd1d899bc
remove overzealous check during HEAD() (#19940)
due to a historic bug in CopyObject() where
an inlined object loses its metadata, the
check causes an incorrect fallback verifying
data-dir.

CopyObject() bug was fixed in ffa91f9794 however
the occurrence of this problem is historic, so
the aforementioned check is stretching too much.

Bonus: simplify fileInfoRaw() to read xl.json as well,
also recreate buckets properly.
2024-06-17 07:29:18 -07:00

192 lines
5.6 KiB
Go

// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"errors"
"github.com/zeebo/xxh3"
)
// getFileInfoVersions partitions this object's versions such that,
// - fivs.Versions has all the non-free versions
// - fivs.FreeVersions has all the free versions
//
// if inclFreeVersions is true all the versions are in fivs.Versions, free and non-free versions alike.
//
// Note: Only the scanner requires fivs.Versions to have exclusively non-free versions. This is used while enforcing NewerNoncurrentVersions lifecycle element.
func getFileInfoVersions(xlMetaBuf []byte, volume, path string, allParts, inclFreeVersions bool) (FileInfoVersions, error) {
fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path, allParts)
if err != nil {
return fivs, err
}
// If inclFreeVersions is false, partition the versions in fivs.Versions
// such that finally fivs.Versions has
// all the non-free versions and fivs.FreeVersions has all the free
// versions.
n := 0
for _, fi := range fivs.Versions {
// filter our tier object delete marker
if fi.TierFreeVersion() {
if !inclFreeVersions {
fivs.FreeVersions = append(fivs.FreeVersions, fi)
}
} else {
if !inclFreeVersions {
fivs.Versions[n] = fi
}
n++
}
}
if !inclFreeVersions {
fivs.Versions = fivs.Versions[:n]
}
// Update numversions
for i := range fivs.Versions {
fivs.Versions[i].NumVersions = n
}
return fivs, nil
}
func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string, allParts bool) (FileInfoVersions, error) {
var versions []FileInfo
var err error
if buf, _, e := isIndexedMetaV2(xlMetaBuf); e != nil {
return FileInfoVersions{}, e
} else if buf != nil {
versions, err = buf.ListVersions(volume, path, allParts)
} else {
var xlMeta xlMetaV2
if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil {
return FileInfoVersions{}, err
}
versions, err = xlMeta.ListVersions(volume, path, allParts)
}
if err == nil && len(versions) == 0 {
// This special case is needed to handle len(xlMeta.versions) == 0
versions = []FileInfo{
{
Volume: volume,
Name: path,
Deleted: true,
IsLatest: true,
ModTime: timeSentinel1970,
},
}
}
if err != nil {
return FileInfoVersions{}, err
}
return FileInfoVersions{
Volume: volume,
Name: path,
Versions: versions,
LatestModTime: versions[0].ModTime,
}, nil
}
type fileInfoOpts struct {
InclFreeVersions bool
Data bool
AllParts bool
}
func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, opts fileInfoOpts) (FileInfo, error) {
var fi FileInfo
var err error
var inData xlMetaInlineData
if buf, data, e := isIndexedMetaV2(xlMetaBuf); e != nil {
return FileInfo{}, e
} else if buf != nil {
inData = data
fi, err = buf.ToFileInfo(volume, path, versionID, opts.AllParts)
if len(buf) != 0 && errors.Is(err, errFileNotFound) {
// This special case is needed to handle len(xlMeta.versions) == 0
return FileInfo{
Volume: volume,
Name: path,
VersionID: versionID,
Deleted: true,
IsLatest: true,
ModTime: timeSentinel1970,
}, nil
}
} else {
var xlMeta xlMetaV2
if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil {
return FileInfo{}, err
}
if len(xlMeta.versions) == 0 {
// This special case is needed to handle len(xlMeta.versions) == 0
return FileInfo{
Volume: volume,
Name: path,
VersionID: versionID,
Deleted: true,
IsLatest: true,
ModTime: timeSentinel1970,
}, nil
}
inData = xlMeta.data
fi, err = xlMeta.ToFileInfo(volume, path, versionID, opts.InclFreeVersions, opts.AllParts)
}
if !opts.Data || err != nil {
return fi, err
}
versionID = fi.VersionID
if versionID == "" {
versionID = nullVersionID
}
fi.Data = inData.find(versionID)
if len(fi.Data) == 0 {
// PR #11758 used DataDir, preserve it
// for users who might have used master
// branch
fi.Data = inData.find(fi.DataDir)
}
return fi, nil
}
// hashDeterministicString will return a deterministic hash for the map values.
// Trivial collisions are avoided, but this is by no means a strong hash.
func hashDeterministicString(m map[string]string) uint64 {
// Seed (random)
crc := uint64(0xc2b40bbac11a7295)
// Xor each value to make order independent
for k, v := range m {
// Separate key and value with an individual xor with a random number.
// Add values of each, so they cannot be trivially collided.
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.HashString(v) ^ 0x8da4c8da66194257)
}
return crc
}
// hashDeterministicBytes will return a deterministic (weak) hash for the map values.
// Trivial collisions are avoided, but this is by no means a strong hash.
func hashDeterministicBytes(m map[string][]byte) uint64 {
crc := uint64(0x1bbc7e1dde654743)
for k, v := range m {
crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.Hash(v) ^ 0x8da4c8da66194257)
}
return crc
}