http: Add Starlark handler

This migrates a feature that was previously reserved for enterprise
users, according to #2786.

The Starlark integration needs to be updated since this was made before
some significant changes in the v2 code base. When functional, it makes
it possible to have very dynamic HTTP handlers. This will be a long-term
ongoing project.

Credit to Danny Navarro

cmd/caddy/main.go

@@ -34,6 +34,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/fastcgi"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
+	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/starlarkmw"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/templates"
 	_ "github.com/caddyserver/caddy/v2/modules/caddytls"
 	_ "github.com/caddyserver/caddy/v2/modules/caddytls/standardstek"

modules/caddyhttp/starlarkmw/example/caddy.json

@@ -0,0 +1,19 @@
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"MY_SERVER": {
+					"listen": [":3001"],
+					"routes": [
+						{
+							"handle": {
+								"handler": "starlark",
+								"script": "def setup(r):\n\t# create some middlewares specific to this request\n\ttemplates = loadModule('http.handlers.templates', {'include_root': './includes'})\n\tmidChain = execute([templates])\n\ndef serveHTTP (rw, r):\n\trw.Write('Hello world, from Starlark!')\n"
+							}
+						}
+					]
+				}
+			}
+		}
+	}
+}

modules/caddyhttp/starlarkmw/internal/lib/module.go

@@ -0,0 +1,165 @@
+package lib
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+
+	"github.com/caddyserver/caddy/v2"
+	"go.starlark.net/starlark"
+)
+
+// ResponderModule represents a module that satisfies the caddyhttp handler.
+type ResponderModule struct {
+	Name     string
+	Cfg      json.RawMessage
+	Instance caddyhttp.Handler
+}
+
+func (r ResponderModule) Freeze()               {}
+func (r ResponderModule) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: responder module") }
+func (r ResponderModule) String() string        { return "responder module" }
+func (r ResponderModule) Type() string          { return "responder module" }
+func (r ResponderModule) Truth() starlark.Bool  { return true }
+
+// Middleware represents a module that satisfies the starlark Value interface.
+type Middleware struct {
+	Name     string
+	Cfg      json.RawMessage
+	Instance caddyhttp.MiddlewareHandler
+}
+
+func (r Middleware) Freeze()               {}
+func (r Middleware) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: middleware") }
+func (r Middleware) String() string        { return "middleware" }
+func (r Middleware) Type() string          { return "middleware" }
+func (r Middleware) Truth() starlark.Bool  { return true }
+
+// LoadMiddleware represents the method exposed to starlark to load a Caddy module.
+type LoadMiddleware struct {
+	Middleware Middleware
+	Ctx        caddy.Context
+}
+
+func (r LoadMiddleware) Freeze()               {}
+func (r LoadMiddleware) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: loadMiddleware") }
+func (r LoadMiddleware) String() string        { return "loadMiddleware" }
+func (r LoadMiddleware) Type() string          { return "function: loadMiddleware" }
+func (r LoadMiddleware) Truth() starlark.Bool  { return true }
+
+// Run is the method bound to the starlark loadMiddleware function.
+func (r *LoadMiddleware) Run(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var name string
+	var cfg *starlark.Dict
+	err := starlark.UnpackPositionalArgs(fn.Name(), args, kwargs, 2, &name, &cfg)
+	if err != nil {
+		return starlark.None, fmt.Errorf("unpacking arguments: %v", err.Error())
+	}
+
+	js := json.RawMessage(cfg.String())
+
+	if strings.Index(name, "http.handlers.") == -1 {
+		name = fmt.Sprintf("http.handlers.%s", name)
+	}
+
+	inst, err := r.Ctx.LoadModule(name, js)
+	if err != nil {
+		return starlark.None, err
+	}
+
+	mid, ok := inst.(caddyhttp.MiddlewareHandler)
+	if !ok {
+		return starlark.None, fmt.Errorf("could not assert as middleware handler")
+	}
+
+	m := Middleware{
+		Name:     name,
+		Cfg:      js,
+		Instance: mid,
+	}
+
+	r.Middleware = m
+
+	return m, nil
+}
+
+// LoadResponder represents the method exposed to starlark to load a Caddy middleware responder.
+type LoadResponder struct {
+	Module ResponderModule
+	Ctx    caddy.Context
+}
+
+func (r LoadResponder) Freeze()               {}
+func (r LoadResponder) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: loadModule") }
+func (r LoadResponder) String() string        { return "loadModule" }
+func (r LoadResponder) Type() string          { return "function: loadModule" }
+func (r LoadResponder) Truth() starlark.Bool  { return true }
+
+// Run is the method bound to the starlark loadResponder function.
+func (r *LoadResponder) Run(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var name string
+	var cfg *starlark.Dict
+	err := starlark.UnpackPositionalArgs(fn.Name(), args, kwargs, 2, &name, &cfg)
+	if err != nil {
+		return starlark.None, fmt.Errorf("unpacking arguments: %v", err.Error())
+	}
+
+	js := json.RawMessage(cfg.String())
+
+	if strings.Index(name, "http.handlers.") == -1 {
+		name = fmt.Sprintf("http.handlers.%s", name)
+	}
+
+	inst, err := r.Ctx.LoadModule(name, js)
+	if err != nil {
+		return starlark.None, err
+	}
+
+	res, ok := inst.(caddyhttp.Handler)
+	if !ok {
+		return starlark.None, fmt.Errorf("could not assert as responder")
+	}
+
+	m := ResponderModule{
+		Name:     name,
+		Cfg:      js,
+		Instance: res,
+	}
+
+	r.Module = m
+
+	return m, nil
+}
+
+// Execute represents the method exposed to starlark to build a middleware chain.
+type Execute struct {
+	Modules []Middleware
+}
+
+func (r Execute) Freeze()               {}
+func (r Execute) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: execute") }
+func (r Execute) String() string        { return "execute" }
+func (r Execute) Type() string          { return "function: execute" }
+func (r Execute) Truth() starlark.Bool  { return true }
+
+// Run is the method bound to the starlark execute function.
+func (r *Execute) Run(thread *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var mids *starlark.List
+	err := starlark.UnpackPositionalArgs(fn.Name(), args, kwargs, 1, &mids)
+	if err != nil {
+		return starlark.None, fmt.Errorf("unpacking arguments: %v", err.Error())
+	}
+
+	for i := 0; i < mids.Len(); i++ {
+		val, ok := mids.Index(i).(Middleware)
+		if !ok {
+			return starlark.None, fmt.Errorf("cannot get module from execute")
+		}
+
+		r.Modules = append(r.Modules, val)
+	}
+
+	return starlark.None, nil
+}

modules/caddyhttp/starlarkmw/starlarkmw.go

@@ -0,0 +1,172 @@
+package starlarkmw
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	caddyscript "github.com/caddyserver/caddy/v2/pkg/caddyscript/lib"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/starlarkmw/internal/lib"
+	"github.com/starlight-go/starlight/convert"
+	"go.starlark.net/starlark"
+)
+
+func init() {
+	caddy.RegisterModule(StarlarkMW{})
+}
+
+// StarlarkMW represents a middleware responder written in starlark
+type StarlarkMW struct {
+	Script         string `json:"script"`
+	serveHTTP      *starlark.Function
+	setup          *starlark.Function
+	thread         *starlark.Thread
+	loadMiddleware *lib.LoadMiddleware
+	execute        *lib.Execute
+	globals        *starlark.StringDict
+	gctx           caddy.Context
+	rctx           caddy.Context
+	rcancel        context.CancelFunc
+}
+
+// CaddyModule returns the Caddy module information.
+func (StarlarkMW) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		Name: "http.handlers.starlark",
+		New:  func() caddy.Module { return new(StarlarkMW) },
+	}
+}
+
+// ServeHTTP responds to an http request with starlark.
+func (s *StarlarkMW) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+	var mwcancel context.CancelFunc
+	var mwctx caddy.Context
+
+	// call setup() to prepare the middleware chain if it is defined
+	if s.setup != nil {
+		mwctx, mwcancel = caddy.NewContext(s.gctx)
+		defer mwcancel()
+
+		s.loadMiddleware.Ctx = mwctx
+		args := starlark.Tuple{caddyscript.HTTPRequest{Req: r}}
+
+		_, err := starlark.Call(new(starlark.Thread), s.setup, args, nil)
+		if err != nil {
+			return fmt.Errorf("starlark setup(), %v", err)
+		}
+	}
+
+	// dynamically build middleware chain for each request
+	stack := caddyhttp.HandlerFunc(func(w http.ResponseWriter, r *http.Request) error {
+		wr, err := convert.ToValue(w)
+		if err != nil {
+			return fmt.Errorf("cannot convert response writer to starlark value")
+		}
+
+		args := starlark.Tuple{wr, caddyscript.HTTPRequest{Req: r}}
+		v, err := starlark.Call(new(starlark.Thread), s.serveHTTP, args, nil)
+		if err != nil {
+			return fmt.Errorf("starlark serveHTTP(), %v", err)
+		}
+
+		// if a responder type was returned from starlark we should run it otherwise it
+		// is expected to handle the request
+		if resp, ok := v.(lib.ResponderModule); ok {
+			return resp.Instance.ServeHTTP(w, r)
+		}
+
+		return nil
+	})
+
+	// TODO :- make middlewareResponseWriter exported and wrap w with that
+	var mid []caddyhttp.Middleware
+	for _, m := range s.execute.Modules {
+		mid = append(mid, func(next caddyhttp.HandlerFunc) caddyhttp.HandlerFunc {
+			return func(w http.ResponseWriter, r *http.Request) error {
+				return m.Instance.ServeHTTP(w, r, next)
+			}
+		})
+	}
+
+	for i := len(mid) - 1; i >= 0; i-- {
+		stack = mid[i](stack)
+	}
+
+	s.execute.Modules = nil
+
+	return stack(w, r)
+}
+
+// Cleanup cleans up any modules loaded during the creation of a starlark route.
+func (s *StarlarkMW) Cleanup() error {
+	s.rcancel()
+	return nil
+}
+
+// Provision sets up the starlark env.
+func (s *StarlarkMW) Provision(ctx caddy.Context) error {
+	// store global context
+	s.gctx = ctx
+
+	// setup context for cleaning up any modules loaded during starlark script parsing phase
+	rctx, cancel := caddy.NewContext(ctx)
+	s.rcancel = cancel
+
+	// setup starlark global env
+	env := make(starlark.StringDict)
+	loadMiddleware := lib.LoadMiddleware{}
+	loadResponder := lib.LoadResponder{
+		Ctx: rctx,
+	}
+	execute := lib.Execute{}
+
+	lr := starlark.NewBuiltin("loadResponder", loadResponder.Run)
+	lr = lr.BindReceiver(&loadResponder)
+	env["loadResponder"] = lr
+
+	lm := starlark.NewBuiltin("loadMiddleware", loadMiddleware.Run)
+	lm = lm.BindReceiver(&loadMiddleware)
+	env["loadMiddleware"] = lm
+
+	ex := starlark.NewBuiltin("execute", execute.Run)
+	ex = ex.BindReceiver(&execute)
+	env["execute"] = ex
+
+	// import caddyscript lib
+	env["time"] = caddyscript.Time{}
+	env["regexp"] = caddyscript.Regexp{}
+
+	// configure starlark
+	thread := new(starlark.Thread)
+	s.thread = thread
+
+	// run starlark script
+	globals, err := starlark.ExecFile(thread, "", s.Script, env)
+	if err != nil {
+		return fmt.Errorf("starlark exec file: %v", err.Error())
+	}
+
+	// extract defined methods to setup middleware chain and responder, setup is not required
+	var setup *starlark.Function
+	if _, ok := globals["setup"]; ok {
+		setup, ok = globals["setup"].(*starlark.Function)
+		if !ok {
+			return fmt.Errorf("setup function not defined in starlark script")
+		}
+	}
+
+	serveHTTP, ok := globals["serveHTTP"].(*starlark.Function)
+	if !ok {
+		return fmt.Errorf("serveHTTP function not defined in starlark script")
+	}
+
+	s.setup = setup
+	s.serveHTTP = serveHTTP
+	s.loadMiddleware = &loadMiddleware
+	s.execute = &execute
+	s.globals = &globals
+
+	return nil
+}

modules/caddyhttp/starlarkmw/tools/gen/example.star

@@ -0,0 +1,40 @@
+# any module that provisions resources
+proxyConfig = {
+	'load_balance_type': 'round_robin',
+	'upstreams': [
+		{
+			'host': 'http://localhost:8080',
+			'circuit_breaker': {
+				'type': 'status_ratio',
+				'threshold': 0.5
+			}
+		},
+		{
+			'host': 'http://localhost:8081'
+		}
+	]
+}
+
+sfConfig = {
+	'root': '/Users/dev/Desktop',
+	'browse': {},
+}
+
+proxy = loadResponder('reverse_proxy', proxyConfig)
+static_files = loadResponder('file_server', sfConfig)
+
+def setup(r):
+	# create some middlewares specific to this request
+	mid = []
+
+	if r.query.get('log') == 'true':
+		logMid = loadMiddleware('log', {'file': 'access.log'})
+		mid.append(logMid)
+
+	execute(mid)
+
+def serveHTTP(w, r):
+	if r.url.find('static') > 0:
+		return static_files
+
+	return proxy