Add detailed parameter tracing + custom prefix (#18518)

* Allow per handler custom prefix.
* Add automatic parameter extraction

cmd/storage-rest-server.go

@@ -732,7 +732,7 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
 	encoder.Encode(dErrsResp)
 }
 
-var storageRenameDataHandler = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenamedata, func() *RenameDataHandlerParams {
+var storageRenameDataHandler = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenameData, func() *RenameDataHandlerParams {
 	return &RenameDataHandlerParams{}
 }, func() *RenameDataResp {
 	return &RenameDataResp{}

internal/grid/connection.go

@@ -336,7 +336,7 @@ func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]by
 		}
 		c.outgoing.Delete(client.MuxID)
 	}()
-	return client.traceRoundtrip(c.trace, h, req)
+	return client.traceRoundtrip(ctx, c.trace, h, req)
 }
 
 // Request allows to do a single remote request.
@@ -364,7 +364,7 @@ func (c *Subroute) Request(ctx context.Context, h HandlerID, req []byte) ([]byte
 		}
 		c.outgoing.Delete(client.MuxID)
 	}()
-	return client.traceRoundtrip(c.trace, h, req)
+	return client.traceRoundtrip(ctx, c.trace, h, req)
 }
 
 // NewStream creates a new stream.

internal/grid/handlers.go

@@ -55,7 +55,7 @@ const (
 	HandlerUpdateMetadata
 	HandlerWriteMetadata
 	HandlerCheckParts
-	HandlerRenamedata
+	HandlerRenameData
 
 	// Add more above here ^^^
 	// If all handlers are used, the type of Handler can be changed.
@@ -65,6 +65,34 @@ const (
 	handlerLast
 )
 
+// handlerPrefixes are prefixes for handler IDs used for tracing.
+// If a handler is not listed here, it will be traced with "grid" prefix.
+var handlerPrefixes = [handlerLast]string{
+	HandlerLockLock:        lockPrefix,
+	HandlerLockRLock:       lockPrefix,
+	HandlerLockUnlock:      lockPrefix,
+	HandlerLockRUnlock:     lockPrefix,
+	HandlerLockRefresh:     lockPrefix,
+	HandlerLockForceUnlock: lockPrefix,
+	HandlerWalkDir:         storagePrefix,
+	HandlerStatVol:         storagePrefix,
+	HandlerDiskInfo:        storagePrefix,
+	HandlerNSScanner:       storagePrefix,
+	HandlerReadXL:          storagePrefix,
+	HandlerReadVersion:     storagePrefix,
+	HandlerDeleteFile:      storagePrefix,
+	HandlerDeleteVersion:   storagePrefix,
+	HandlerUpdateMetadata:  storagePrefix,
+	HandlerWriteMetadata:   storagePrefix,
+	HandlerCheckParts:      storagePrefix,
+	HandlerRenameData:      storagePrefix,
+}
+
+const (
+	lockPrefix    = "lock"
+	storagePrefix = "storageR"
+)
+
 func init() {
 	// Static check if we exceed 255 handler ids.
 	// Extend the type to uint16 when hit.
@@ -364,6 +392,7 @@ func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Re
 	if err != nil {
 		return resp, err
 	}
+	ctx = context.WithValue(ctx, TraceParamsKey{}, req)
 	res, err := c.Request(ctx, h.id, payload)
 	PutByteBuffer(payload)
 	if err != nil {

internal/grid/handlers_string.go

@@ -26,13 +26,13 @@ func _() {
 	_ = x[HandlerUpdateMetadata-15]
 	_ = x[HandlerWriteMetadata-16]
 	_ = x[HandlerCheckParts-17]
-	_ = x[HandlerRenamedata-18]
+	_ = x[HandlerRenameData-18]
 	_ = x[handlerTest-19]
 	_ = x[handlerTest2-20]
 	_ = x[handlerLast-21]
 }
 
-const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenamedatahandlerTesthandlerTest2handlerLast"
+const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDatahandlerTesthandlerTest2handlerLast"
 
 var _HandlerID_index = [...]uint8{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 207, 219, 230}
 

internal/grid/trace.go

@@ -18,19 +18,27 @@
 package grid
 
 import (
+	"context"
+	"fmt"
 	"net/http"
+	"net/url"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/pubsub"
 )
 
+// TraceParamsKey allows to pass trace parameters to the request via context.
+// This is only needed when un-typed requests are used.
+// MSS, map[string]string types are preferred, but any struct with exported fields will work.
+type TraceParamsKey struct{}
+
 // traceRequests adds request tracing to the connection.
 func (c *Connection) traceRequests(p *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
 	c.trace = &tracer{
 		Publisher: p,
 		TraceType: madmin.TraceInternal,
-		Prefix:    "grid.",
+		Prefix:    "grid",
 		Local:     c.Local,
 		Remote:    c.Remote,
 		Subroute:  "",
@@ -56,7 +64,7 @@ type tracer struct {
 	Subroute  string
 }
 
-func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte, error) {
+func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID, req []byte) ([]byte, error) {
 	if t == nil || t.Publisher.NumSubscribers(t.TraceType) == 0 {
 		return c.roundtrip(h, req)
 	}
@@ -74,9 +82,14 @@ func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte,
 			status = http.StatusBadRequest
 		}
 	}
+
+	prefix := t.Prefix
+	if p := handlerPrefixes[h]; p != "" {
+		prefix = p
+	}
 	trace := madmin.TraceInfo{
 		TraceType: t.TraceType,
-		FuncName:  t.Prefix + h.String(),
+		FuncName:  prefix + "." + h.String(),
 		NodeName:  t.Local,
 		Time:      start,
 		Duration:  end.Sub(start),
@@ -105,6 +118,20 @@ func (c *muxClient) traceRoundtrip(t *tracer, h HandlerID, req []byte) ([]byte,
 			},
 		},
 	}
+	// If the context contains a TraceParamsKey, add it to the trace path.
+	v := ctx.Value(TraceParamsKey{})
+	if p, ok := v.(*MSS); ok && p != nil {
+		trace.Path += p.ToQuery()
+		trace.HTTP.ReqInfo.Path = trace.Path
+	} else if p, ok := v.(map[string]string); ok {
+		m := MSS(p)
+		trace.Path += m.ToQuery()
+		trace.HTTP.ReqInfo.Path = trace.Path
+	} else if v != nil {
+		// Print exported fields as single request to path.
+		trace.Path = fmt.Sprintf("%s?req=%s", trace.Path, url.QueryEscape(fmt.Sprintf("%+v", v)))
+		trace.HTTP.ReqInfo.Path = trace.Path
+	}
 	t.Publisher.Publish(trace)
 	return resp, err
 }

internal/grid/types.go

@@ -19,6 +19,9 @@ package grid
 
 import (
 	"errors"
+	"net/url"
+	"sort"
+	"strings"
 
 	"github.com/tinylib/msgp/msgp"
 )
@@ -117,6 +120,31 @@ func NewMSSWith(m map[string]string) *MSS {
 	return &m2
 }
 
+// ToQuery constructs a URL query string from the MSS, including "?" if there are any keys.
+func (m MSS) ToQuery() string {
+	if len(m) == 0 {
+		return ""
+	}
+	var buf strings.Builder
+	buf.WriteByte('?')
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		v := m[k]
+		keyEscaped := url.QueryEscape(k)
+		if buf.Len() > 1 {
+			buf.WriteByte('&')
+		}
+		buf.WriteString(keyEscaped)
+		buf.WriteByte('=')
+		buf.WriteString(url.QueryEscape(v))
+	}
+	return buf.String()
+}
+
 // NewBytes returns a new Bytes.
 func NewBytes() *Bytes {
 	b := Bytes(GetByteBuffer()[:0])