self-hosted/minio
self-hosted/minio
1
0
Fork 0
mirror of https://github.com/minio/minio synced 2024-10-14 11:52:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

allow more workers for ILM expiration (#16924)

Browse source
This commit is contained in:
Harshavardhana 2023-03-30 10:47:15 -07:00 committed by GitHub
parent c468b4e2a8
commit 7a6c4e438e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
cmd/bucket-lifecycle.go
View file

@ -24,6 +24,8 @@ import (
"fmt"
"io"
"net/http"
"runtime"
"strconv"
"strings"
"sync"
"sync/atomic"
@ -38,6 +40,8 @@ import (
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/s3select"
"github.com/minio/minio/internal/workers"
"github.com/minio/pkg/env"
)
const (
@ -116,26 +120,49 @@ var globalExpiryState *expiryState
func newExpiryState() *expiryState {
return &expiryState{
byDaysCh: make(chan expiryTask, 10000),
byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
byDaysCh: make(chan expiryTask, 100000),
byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
}
}
func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
globalExpiryState = newExpiryState()
workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
ewk, err := workers.New(workerSize)
if err != nil {
logger.LogIf(ctx, err)
}
nwk, err := workers.New(workerSize)
if err != nil {
logger.LogIf(ctx, err)
}
go func() {
for t := range globalExpiryState.byDaysCh {
if t.objInfo.TransitionedObject.Status != "" {
applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
} else {
applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
}
ewk.Take()
go func(t expiryTask) {
defer ewk.Give()
if t.objInfo.TransitionedObject.Status != "" {
applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
} else {
applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
}
}(t)
}
ewk.Wait()
}()
go func() {
for t := range globalExpiryState.byNewerNoncurrentCh {
deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
nwk.Take()
go func(t newerNoncurrentTask) {
defer nwk.Give()
deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
}(t)
}
nwk.Wait()
}()
}