Add a warning when the total size of an object versions exceeds 1 TiB (#19435)

cmd/data-scanner.go

@@ -62,11 +62,12 @@ var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper              = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
-	scannerCycle                = uatomic.NewDuration(dataScannerStartDelay)
-	scannerIdleMode             = uatomic.NewInt32(0) // default is throttled when idle
-	scannerExcessObjectVersions = uatomic.NewInt64(100)
-	scannerExcessFolders        = uatomic.NewInt64(50000)
+	scannerSleeper                       = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
+	scannerCycle                         = uatomic.NewDuration(dataScannerStartDelay)
+	scannerIdleMode                      = uatomic.NewInt32(0) // default is throttled when idle
+	scannerExcessObjectVersions          = uatomic.NewInt64(100)
+	scannerExcessObjectVersionsTotalSize = uatomic.NewInt64(1024 * 1024 * 1024 * 1024) // 1 TB
+	scannerExcessFolders                 = uatomic.NewInt64(50000)
 )
 
 // initDataScanner will start the scanner in the background.
@@ -1065,7 +1066,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(objInfos) > int(scannerExcessObjectVersions.Load()) {
+	if len(objInfos) >= int(scannerExcessObjectVersions.Load()) {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1089,6 +1090,39 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 		})
 	}
 
+	cumulativeSize := int64(0)
+	for _, objInfo := range objInfos {
+		cumulativeSize += objInfo.Size
+	}
+	// Check if the cumulative size of all versions of this object is high.
+	if cumulativeSize >= scannerExcessObjectVersionsTotalSize.Load() {
+		// Notify object accessed via a GET request.
+		sendEvent(eventArgs{
+			EventName:  event.ObjectLargeVersions,
+			BucketName: i.bucket,
+			Object: ObjectInfo{
+				Name: i.objectPath(),
+			},
+			UserAgent: "Scanner",
+			Host:      globalLocalNodeName,
+			RespElements: map[string]string{
+				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
+				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
+			},
+		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:   "scanner:largeversions",
+			APIName: "Scanner",
+			Bucket:  i.bucket,
+			Object:  i.objectPath(),
+			Tags: map[string]interface{}{
+				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
+				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
+			},
+		})
+	}
+
 	return objInfos, nil
 }
 

internal/event/name.go

@@ -61,6 +61,7 @@ const (
 	ObjectTransitionFailed
 	ObjectTransitionComplete
 	ObjectManyVersions
+	ObjectLargeVersions
 	PrefixManyFolders
 
 	objectSingleTypesEnd
@@ -124,6 +125,7 @@ func (name Name) Expand() []Name {
 	case ObjectScannerAll:
 		return []Name{
 			ObjectManyVersions,
+			ObjectLargeVersions,
 			PrefixManyFolders,
 		}
 	case Everything:
@@ -223,6 +225,9 @@ func (name Name) String() string {
 		return "s3:ObjectTransition:Complete"
 	case ObjectManyVersions:
 		return "s3:Scanner:ManyVersions"
+	case ObjectLargeVersions:
+		return "s3:Scanner:LargeVersions"
+
 	case PrefixManyFolders:
 		return "s3:Scanner:BigPrefix"
 	}
@@ -345,6 +350,8 @@ func ParseName(s string) (Name, error) {
 		return ObjectTransitionAll, nil
 	case "s3:Scanner:ManyVersions":
 		return ObjectManyVersions, nil
+	case "s3:Scanner:LargeVersions":
+		return ObjectLargeVersions, nil
 	case "s3:Scanner:BigPrefix":
 		return PrefixManyFolders, nil
 	default: