Use tracing handler per server not per route (#16342)

Tracing handlers consume a large amount of memory over time, which
makes using one per route suboptimal (See https://github.com/open-telemetry/opentelemetry-go-contrib/issues/2413).
Wrapping all `http.Server` handlers with a tracing handler provides
the same tracing abilities for a fraction of the memory usage. A new
span formatter is added which allows servers to still contain the
request span in the span name.

api/observability/tracing/tracing.go

@@ -51,3 +51,10 @@ func DefaultProvider() oteltrace.TracerProvider {
 func HTTPTransportFormatter(_ string, r *http.Request) string {
 	return "HTTP " + r.Method + " " + r.URL.Path
 }
+
+// HTTPHandlerFormatter is a span formatter that may be provided to
+// otelhttp.WithSpanNameFormatter to include the component and url path in the span
+// names generated by otelhttp.NewHandler
+func HTTPHandlerFormatter(operation string, r *http.Request) string {
+	return operation + " " + r.Method + " " + r.URL.Path
+}

lib/auth/middleware.go

@@ -30,6 +30,7 @@ import (
 	"github.com/gravitational/teleport/api/types"
 	apiutils "github.com/gravitational/teleport/api/utils"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/httplib"
 	"github.com/gravitational/teleport/lib/limiter"
 	"github.com/gravitational/teleport/lib/multiplexer"
 	"github.com/gravitational/teleport/lib/observability/metrics"
@@ -166,7 +167,7 @@ func NewTLSServer(cfg TLSServerConfig) (*TLSServer, error) {
 	server := &TLSServer{
 		cfg: cfg,
 		httpServer: &http.Server{
-			Handler:           limiter,
+			Handler:           httplib.MakeTracingHandler(limiter, teleport.ComponentAuth),
 			ReadHeaderTimeout: apidefaults.DefaultDialTimeout,
 		},
 		log: logrus.WithFields(logrus.Fields{

lib/httplib/httplib.go

@@ -29,6 +29,7 @@ import (
 	"strconv"
 
 	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/observability/tracing"
 	"github.com/gravitational/teleport/lib/httplib/csrf"
 	"github.com/gravitational/teleport/lib/utils"
 
@@ -55,19 +56,13 @@ type ErrorWriter func(w http.ResponseWriter, err error)
 
 // MakeHandler returns a new httprouter.Handle func from a handler func
 func MakeHandler(fn HandlerFunc) httprouter.Handle {
-	return MakeTracingHandler(MakeHandlerWithErrorWriter(fn, trace.WriteError))
+	return MakeHandlerWithErrorWriter(fn, trace.WriteError)
 }
 
 // MakeTracingHandler returns a new httprouter.Handle func that wraps the provided handler func
 // with one that will add a tracing span for each request.
-func MakeTracingHandler(h httprouter.Handle) httprouter.Handle {
-	return func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
-		handler := otelhttp.NewHandler(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-			h(writer, request, p)
-		}), r.RequestURI)
-
-		handler.ServeHTTP(w, r)
-	}
+func MakeTracingHandler(h http.Handler, component string) http.Handler {
+	return otelhttp.NewHandler(h, component, otelhttp.WithSpanNameFormatter(tracing.HTTPHandlerFormatter))
 }
 
 // MakeHandlerWithErrorWriter returns a httprouter.Handle from the HandlerFunc,

lib/kube/proxy/server.go

@@ -28,6 +28,7 @@ import (
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/httplib"
 	"github.com/gravitational/teleport/lib/limiter"
 	"github.com/gravitational/teleport/lib/multiplexer"
 	"github.com/gravitational/teleport/lib/reversetunnel"
@@ -136,7 +137,7 @@ func NewTLSServer(cfg TLSServerConfig) (*TLSServer, error) {
 		fwd:             fwd,
 		TLSServerConfig: cfg,
 		Server: &http.Server{
-			Handler:           limiter,
+			Handler:           httplib.MakeTracingHandler(limiter, teleport.ComponentKube),
 			ReadHeaderTimeout: apidefaults.DefaultDialTimeout * 2,
 			TLSConfig:         cfg.TLS,
 		},

lib/service/service.go

@@ -82,6 +82,7 @@ import (
 	"github.com/gravitational/teleport/lib/events/firestoreevents"
 	"github.com/gravitational/teleport/lib/events/gcssessions"
 	"github.com/gravitational/teleport/lib/events/s3sessions"
+	"github.com/gravitational/teleport/lib/httplib"
 	"github.com/gravitational/teleport/lib/inventory"
 	"github.com/gravitational/teleport/lib/joinserver"
 	kubeproxy "github.com/gravitational/teleport/lib/kube/proxy"
@@ -3476,7 +3477,7 @@ func (process *TeleportProcess) initProxyEndpoint(conn *Connector) error {
 		}
 
 		webServer = &http.Server{
-			Handler:           proxyLimiter,
+			Handler:           httplib.MakeTracingHandler(proxyLimiter, teleport.ComponentProxy),
 			ReadHeaderTimeout: apidefaults.DefaultDialTimeout,
 			ErrorLog:          utils.NewStdlogger(log.Error, teleport.ComponentProxy),
 		}
@@ -3966,7 +3967,7 @@ func (process *TeleportProcess) initMinimalReverseTunnel(listeners *proxyListene
 	})
 
 	minimalWebServer = &http.Server{
-		Handler:           minimalProxyLimiter,
+		Handler:           httplib.MakeTracingHandler(minimalProxyLimiter, teleport.ComponentProxy),
 		ReadHeaderTimeout: apidefaults.DefaultDialTimeout,
 		ErrorLog:          utils.NewStdlogger(log.Error, teleport.ComponentReverseTunnelServer),
 	}

lib/srv/app/server.go

@@ -35,6 +35,7 @@ import (
 	apiutils "github.com/gravitational/teleport/api/utils"
 	"github.com/gravitational/teleport/lib/auth"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/httplib"
 	"github.com/gravitational/teleport/lib/labels"
 	"github.com/gravitational/teleport/lib/reversetunnel"
 	"github.com/gravitational/teleport/lib/services"
@@ -843,7 +844,7 @@ func (s *Server) newHTTPServer() *http.Server {
 	s.authMiddleware.Wrap(s)
 
 	return &http.Server{
-		Handler:           s.authMiddleware,
+		Handler:           httplib.MakeTracingHandler(s.authMiddleware, teleport.ComponentApp),
 		ReadHeaderTimeout: apidefaults.DefaultDialTimeout,
 		ErrorLog:          utils.NewStdlogger(s.log.Error, teleport.ComponentApp),
 	}