mirror of
https://github.com/minio/minio
synced 2024-07-20 09:57:03 +00:00
9693c382a8
instead upon any error in renameData(), we still preserve the existing dataDir in some form for recoverability in strange situations such as out of disk space type errors. Bonus: avoid running list and heal() instead allow versions disparity to return the actual versions, uuid to heal. Currently limit this to 100 versions and lesser disparate objects. an undo now reverts back the xl.meta from xl.meta.bkp during overwrites on such flaky setups. Bonus: Save N depth syscalls via skipping the parents upon overwrites and versioned updates. Flaky setup examples are stretch clusters with regular packet drops etc, we need to add some defensive code around to avoid dangling objects.
131 lines
3.2 KiB
Go
131 lines
3.2 KiB
Go
// Copyright (c) 2015-2021 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as published by
|
|
// the Free Software Foundation, either version 3 of the License, or
|
|
// (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/minio/madmin-go/v3"
|
|
"github.com/minio/pkg/v2/wildcard"
|
|
)
|
|
|
|
const (
|
|
mrfOpsQueueSize = 100000
|
|
)
|
|
|
|
// partialOperation is a successful upload/delete of an object
|
|
// but not written in all disks (having quorum)
|
|
type partialOperation struct {
|
|
bucket string
|
|
object string
|
|
versionID string
|
|
versions []byte
|
|
setIndex, poolIndex int
|
|
queued time.Time
|
|
scanMode madmin.HealScanMode
|
|
}
|
|
|
|
// mrfState sncapsulates all the information
|
|
// related to the global background MRF.
|
|
type mrfState struct {
|
|
opCh chan partialOperation
|
|
}
|
|
|
|
// Add a partial S3 operation (put/delete) when one or more disks are offline.
|
|
func (m *mrfState) addPartialOp(op partialOperation) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
select {
|
|
case m.opCh <- op:
|
|
default:
|
|
}
|
|
}
|
|
|
|
var healSleeper = newDynamicSleeper(5, time.Second, false)
|
|
|
|
// healRoutine listens to new disks reconnection events and
|
|
// issues healing requests for queued objects belonging to the
|
|
// corresponding erasure set
|
|
func (m *mrfState) healRoutine(z *erasureServerPools) {
|
|
for {
|
|
select {
|
|
case <-GlobalContext.Done():
|
|
return
|
|
case u, ok := <-m.opCh:
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
// We might land at .metacache, .trash, .multipart
|
|
// no need to heal them skip, only when bucket
|
|
// is '.minio.sys'
|
|
if u.bucket == minioMetaBucket {
|
|
// No MRF needed for temporary objects
|
|
if wildcard.Match("buckets/*/.metacache/*", u.object) {
|
|
continue
|
|
}
|
|
if wildcard.Match("tmp/*", u.object) {
|
|
continue
|
|
}
|
|
if wildcard.Match("multipart/*", u.object) {
|
|
continue
|
|
}
|
|
if wildcard.Match("tmp-old/*", u.object) {
|
|
continue
|
|
}
|
|
}
|
|
|
|
now := time.Now()
|
|
if now.Sub(u.queued) < time.Second {
|
|
// let recently failed networks to reconnect
|
|
// making MRF wait for 1s before retrying,
|
|
// i.e 4 reconnect attempts.
|
|
time.Sleep(time.Second)
|
|
}
|
|
|
|
// wait on timer per heal
|
|
wait := healSleeper.Timer(context.Background())
|
|
|
|
scan := madmin.HealNormalScan
|
|
if u.scanMode != 0 {
|
|
scan = u.scanMode
|
|
}
|
|
if u.object == "" {
|
|
healBucket(u.bucket, scan)
|
|
} else {
|
|
if len(u.versions) > 0 {
|
|
vers := len(u.versions) / 16
|
|
if vers > 0 {
|
|
for i := 0; i < vers; i++ {
|
|
healObject(u.bucket, u.object, uuid.UUID(u.versions[16*i:]).String(), scan)
|
|
}
|
|
}
|
|
} else {
|
|
healObject(u.bucket, u.object, u.versionID, scan)
|
|
}
|
|
}
|
|
|
|
wait()
|
|
}
|
|
}
|
|
}
|