diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 430486f57..2e6a86d17 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -786,6 +786,31 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum) } +// deleteEmptyDir knows only how to remove an empty directory (not the empty object with a +// trailing slash), this is called for the healing code to remove such directories. +func (er erasureObjects) deleteEmptyDir(ctx context.Context, bucket, object string) error { + defer ObjectPathUpdated(pathJoin(bucket, object)) + + if bucket == minioMetaTmpBucket { + return nil + } + + disks := er.getDisks() + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + index := index + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + return disks[index].Delete(ctx, bucket, object, false) + }, index) + } + + // return errors if any during deletion + return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, len(disks)/2+1) +} + // deleteObject - wrapper for delete object, deletes an object from // all the disks in parallel, including `xl.meta` associated with the // object. diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 9a1a9fc77..f4192597a 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1359,6 +1359,14 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str break } + // Remove empty directories if found - they have no meaning. + // Can be left over from highly concurrent put/remove. + if quorumCount > set.setDriveCount/2 && entry.IsEmptyDir { + if !opts.DryRun && opts.Remove { + set.deleteEmptyDir(ctx, bucket, entry.Name) + } + } + // Indicate that first attempt was a success and subsequent loop // knows that its not our first attempt at 'prefix' err = nil diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go index 5cce92a33..f723ec1cc 100644 --- a/cmd/storage-datatypes.go +++ b/cmd/storage-datatypes.go @@ -76,6 +76,8 @@ type FileInfoVersions struct { // Name of the file. Name string + IsEmptyDir bool + // Represents the latest mod time of the // latest version. LatestModTime time.Time diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go index 4dd1e686a..6795e2a50 100644 --- a/cmd/storage-datatypes_gen.go +++ b/cmd/storage-datatypes_gen.go @@ -737,6 +737,12 @@ func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "Name") return } + case "IsEmptyDir": + z.IsEmptyDir, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsEmptyDir") + return + } case "LatestModTime": z.LatestModTime, err = dc.ReadTime() if err != nil { @@ -775,9 +781,9 @@ func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 4 + // map header, size 5 // write "Volume" - err = en.Append(0x84, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + err = en.Append(0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) if err != nil { return } @@ -796,6 +802,16 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Name") return } + // write "IsEmptyDir" + err = en.Append(0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.IsEmptyDir) + if err != nil { + err = msgp.WrapError(err, "IsEmptyDir") + return + } // write "LatestModTime" err = en.Append(0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) if err != nil { @@ -829,13 +845,16 @@ func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { // MarshalMsg implements msgp.Marshaler func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) - // map header, size 4 + // map header, size 5 // string "Volume" - o = append(o, 0x84, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) + o = append(o, 0x85, 0xa6, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65) o = msgp.AppendString(o, z.Volume) // string "Name" o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) o = msgp.AppendString(o, z.Name) + // string "IsEmptyDir" + o = append(o, 0xaa, 0x49, 0x73, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x44, 0x69, 0x72) + o = msgp.AppendBool(o, z.IsEmptyDir) // string "LatestModTime" o = append(o, 0xad, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) o = msgp.AppendTime(o, z.LatestModTime) @@ -882,6 +901,12 @@ func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Name") return } + case "IsEmptyDir": + z.IsEmptyDir, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsEmptyDir") + return + } case "LatestModTime": z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts) if err != nil { @@ -921,7 +946,7 @@ func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *FileInfoVersions) Msgsize() (s int) { - s = 1 + 7 + msgp.StringPrefixSize + len(z.Volume) + 5 + msgp.StringPrefixSize + len(z.Name) + 14 + msgp.TimeSize + 9 + msgp.ArrayHeaderSize + s = 1 + 7 + msgp.StringPrefixSize + len(z.Volume) + 5 + msgp.StringPrefixSize + len(z.Name) + 11 + msgp.BoolSize + 14 + msgp.TimeSize + 9 + msgp.ArrayHeaderSize for za0001 := range z.Versions { s += z.Versions[za0001].Msgsize() } diff --git a/cmd/tree-walk.go b/cmd/tree-walk.go index 21003f149..ff967bef3 100644 --- a/cmd/tree-walk.go +++ b/cmd/tree-walk.go @@ -24,8 +24,9 @@ import ( // TreeWalkResult - Tree walk result carries results of tree walking. type TreeWalkResult struct { - entry string - end bool + entry string + isEmptyDir bool + end bool } // Return entries that have prefix prefixEntry. @@ -254,7 +255,7 @@ func doTreeWalk(ctx context.Context, bucket, prefixDir, entryPrefixMatch, marker select { case <-endWalkCh: return false, errWalkAbort - case resultCh <- TreeWalkResult{entry: pathJoin(prefixDir, entry), end: isEOF}: + case resultCh <- TreeWalkResult{entry: pathJoin(prefixDir, entry), isEmptyDir: leafDir, end: isEOF}: } } diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 0cf7c4c62..2e6de27ba 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -811,8 +811,9 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st var fiv FileInfoVersions if HasSuffix(walkResult.entry, SlashSeparator) { fiv = FileInfoVersions{ - Volume: volume, - Name: walkResult.entry, + Volume: volume, + Name: walkResult.entry, + IsEmptyDir: walkResult.isEmptyDir, Versions: []FileInfo{ { Volume: volume,