Add prefixes usage in Accounting Usage Info (#12687)

cmd/admin-handlers.go

@@ -344,6 +344,40 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
+// PrefixUsageInfoHandler - GET /minio/admin/v3/prefixusage
+// ----------
+// Get server/cluster data usage info
+func (a adminAPIHandlers) PrefixUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PrefixUsageInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	bucket := r.URL.Query().Get("bucket")
+	if isReservedOrInvalidBucket(bucket, false) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
+		return
+	}
+
+	prefixUsageInfo, err := loadPrefixUsageFromBackend(ctx, objectAPI, bucket)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	prefixUsageInfoJSON, err := json.Marshal(prefixUsageInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, prefixUsageInfoJSON)
+}
+
 func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,

cmd/admin-router.go

@@ -72,6 +72,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
 		// DataUsageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
+		// PrefixUsageInfo operations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/prefixusageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.PrefixUsageInfoHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
 			/// Heal operations

cmd/data-usage-cache.go

@@ -530,6 +530,18 @@ func (h dataUsageHash) Key() string {
 	return string(h)
 }
 
+func (d *dataUsageCache) flattenChildrens(root dataUsageEntry) (m map[string]dataUsageEntry) {
+	m = make(map[string]dataUsageEntry)
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		m[id] = e
+	}
+	return m
+}
+
 // flatten all children of the root into the root element and return it.
 func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
 	for id := range root.Children {

cmd/data-usage.go

@@ -20,7 +20,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"net/http"
+	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
@@ -59,6 +61,39 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 	}
 }
 
+// loadPrefixUsageFromBackend returns prefix usages found in passed buckets
+//   e.g.:  /testbucket/prefix => 355601334
+func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) {
+	z, ok := objAPI.(*erasureServerPools)
+	if !ok {
+		return nil, errors.New("prefix usage is not supported")
+	}
+
+	cache := dataUsageCache{}
+
+	m := make(map[string]uint64)
+	for _, pool := range z.serverPools {
+		for _, er := range pool.sets {
+			// Load bucket usage prefixes
+			if err := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName); err == nil {
+				root := cache.find(bucket)
+				if root == nil {
+					// We dont have usage information for this bucket in this
+					// set, go to the next set
+					continue
+				}
+
+				for id, usageInfo := range cache.flattenChildrens(*root) {
+					prefix := strings.TrimPrefix(id, bucket+slashSeparator)
+					m[prefix] += uint64(usageInfo.Size)
+				}
+			}
+		}
+	}
+
+	return m, nil
+}
+
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.DataUsageInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {