diff --git a/Makefile b/Makefile index 54f41b4ad25..6903c219852 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ # Naming convention: # for stable releases we use "1.0.0" format # for pre-releases, we use "1.0.0-beta.2" format -VERSION=4.2.0-alpha.5 +VERSION=4.2.0-dev.4 DOCKER_IMAGE ?= quay.io/gravitational/teleport @@ -45,10 +45,17 @@ PAM_TAG := pam PAM_MESSAGE := "with PAM support" endif +# BPF support will only be built into Teleport if headers exist at build time. +BPF_MESSAGE := "without BPF support" +ifneq ("$(wildcard /usr/include/bcc/libbpf.h)","") +BPF_TAG := bpf +BPF_MESSAGE := "with BPF support" +endif + # On Windows only build tsh. On all other platforms build teleport, tctl, # and tsh. BINARIES=$(BUILDDIR)/teleport $(BUILDDIR)/tctl $(BUILDDIR)/tsh -RELEASE_MESSAGE := "Building with GOOS=$(OS) GOARCH=$(ARCH) and $(PAM_MESSAGE) and $(FIPS_MESSAGE)." +RELEASE_MESSAGE := "Building with GOOS=$(OS) GOARCH=$(ARCH) and $(PAM_MESSAGE) and $(FIPS_MESSAGE) and $(BPF_MESSAGE)." ifeq ("$(OS)","windows") BINARIES=$(BUILDDIR)/tsh endif @@ -78,11 +85,11 @@ all: $(VERSRC) # If you are considering changing this behavior, please consult with dev team first .PHONY: $(BUILDDIR)/tctl $(BUILDDIR)/tctl: - GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG)" -o $(BUILDDIR)/tctl $(BUILDFLAGS) ./tool/tctl + GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" -o $(BUILDDIR)/tctl $(BUILDFLAGS) ./tool/tctl .PHONY: $(BUILDDIR)/teleport $(BUILDDIR)/teleport: - GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport + GOOS=$(OS) GOARCH=$(ARCH) $(CGOFLAG) go build -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" -o $(BUILDDIR)/teleport $(BUILDFLAGS) ./tool/teleport .PHONY: $(BUILDDIR)/tsh $(BUILDDIR)/tsh: @@ -188,7 +195,7 @@ run-docs: .PHONY: test test: FLAGS ?= test: $(VERSRC) - go test ./tool/tsh/... \ + go test -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" ./tool/tsh/... \ ./lib/... \ ./tool/teleport... $(FLAGS) $(ADDFLAGS) go vet ./tool/... ./lib/... @@ -199,7 +206,7 @@ test: $(VERSRC) .PHONY: integration integration: @echo KUBECONFIG is: $(KUBECONFIG), TEST_KUBE: $(TEST_KUBE) - go test -tags "$(PAM_TAG) $(FIPS_TAG)" ./integration/... + go test -v -tags "$(PAM_TAG) $(FIPS_TAG) $(BPF_TAG)" ./integration/... # This rule triggers re-generation of version.go and gitref.go if Makefile changes $(VERSRC): Makefile diff --git a/build.assets/Makefile b/build.assets/Makefile index 2686b637258..48780ba718c 100644 --- a/build.assets/Makefile +++ b/build.assets/Makefile @@ -7,7 +7,8 @@ DOCSDIR=/teleport HOSTNAME=buildbox SRCDIR=/gopath/src/github.com/gravitational/teleport -DOCKERFLAGS := --rm=true -v "$$(pwd)/../":$(SRCDIR) -v /tmp:/tmp -v "$$(pwd)/bcc:/usr/include/bcc" -w $(SRCDIR) -h $(HOSTNAME) +DOCKERFLAGS := --rm=true -v "$$(pwd)/../":$(SRCDIR) -v /tmp:/tmp -w $(SRCDIR) -h $(HOSTNAME) +BCCFLAGS := -v "$$(pwd)/bcc:/usr/include/bcc" ADDFLAGS=-ldflags -w NOROOT=-u $$(id -u):$$(id -g) KUBECONFIG ?= @@ -33,7 +34,7 @@ export # .PHONY:build build: bbox - docker run $(DOCKERFLAGS) $(NOROOT) $(BBOX) \ + docker run $(DOCKERFLAGS) $(BCCFLAGS) $(NOROOT) $(BBOX) \ make -C $(SRCDIR) ADDFLAGS='$(ADDFLAGS)' release # @@ -41,7 +42,7 @@ build: bbox # .PHONY:build-binaries build-binaries: bbox - docker run $(DOCKERFLAGS) $(NOROOT) $(BBOX) \ + docker run $(DOCKERFLAGS) $(BCCFLAGS) $(NOROOT) $(BBOX) \ make -C $(SRCDIR) ADDFLAGS='$(ADDFLAGS)' all # @@ -166,7 +167,7 @@ run-docs: docsbox # .PHONY:enter enter: bbox - docker run $(DOCKERFLAGS) -ti $(NOROOT) \ + docker run $(DOCKERFLAGS) $(BCCFLAGS) -ti $(NOROOT) \ -e HOME=$(SRCDIR)/build.assets -w $(SRCDIR) $(BBOX) /bin/bash # @@ -174,7 +175,7 @@ enter: bbox # .PHONY:release release: bbox - docker run $(DOCKERFLAGS) -i $(NOROOT) $(BBOX) \ + docker run $(DOCKERFLAGS) $(BCCFLAGS) -i $(NOROOT) $(BBOX) \ /usr/bin/make release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(RUNTIME) # @@ -184,7 +185,7 @@ release: bbox .PHONY:release-fips release-fips: bbox-fips @if [ -z ${VERSION} ]; then echo "VERSION is not set"; exit 1; fi - docker run $(DOCKERFLAGS) -i $(NOROOT) $(BBOXFIPS) \ + docker run $(DOCKERFLAGS) $(BCCFLAGS) -i $(NOROOT) $(BBOXFIPS) \ /usr/bin/make -C e release -e ADDFLAGS="$(ADDFLAGS)" OS=$(OS) ARCH=$(ARCH) RUNTIME=$(RUNTIME) FIPS=yes VERSION=$(VERSION) GITTAG=v$(VERSION) # diff --git a/constants.go b/constants.go index 250cb46c4d6..bb8646f3874 100644 --- a/constants.go +++ b/constants.go @@ -206,6 +206,12 @@ const ( // used to broadcast events to subscribers. ComponentBuffer = "buffer" + // ComponentBPF is the eBPF packagae. + ComponentBPF = "bpf" + + // ComponentCgroup is the cgroup package. + ComponentCgroup = "cgroups" + // DebugEnvVar tells tests to use verbose debug output DebugEnvVar = "DEBUG" @@ -560,5 +566,27 @@ const ( OpenBrowserWindows = "rundll32.exe" ) +const ( + // EnhancedRecordingMinKernel is the minimum kernel version for the enhanced + // recording feature. + EnhancedRecordingMinKernel = "4.18.0" + + // EnhancedRecordingCommand is a role option that implies command events are + // captured. + EnhancedRecordingCommand = "command" + + // EnhancedRecordingDisk is a role option that implies disk events are captured. + EnhancedRecordingDisk = "disk" + + // EnhancedRecordingNetwork is a role option that implies network events + // are captured. + EnhancedRecordingNetwork = "network" +) + +const ( + // ExecSubCommand is the sub-command Teleport uses to re-exec itself. + ExecSubCommand = "exec" +) + // RSAKeySize is the size of the RSA key. const RSAKeySize = 2048 diff --git a/e b/e index 2851453a7ba..821f9d64eb2 160000 --- a/e +++ b/e @@ -1 +1 @@ -Subproject commit 2851453a7bafabc9442f9be4ece8aa92b86044a2 +Subproject commit 821f9d64eb2800d3e9fe8a511a05fbdbca5c57b1 diff --git a/integration/integration_test.go b/integration/integration_test.go index 5bc45aff842..fb9b457067a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -45,6 +45,7 @@ import ( "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/auth/testauthority" "github.com/gravitational/teleport/lib/backend" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" @@ -53,6 +54,7 @@ import ( "github.com/gravitational/teleport/lib/service" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/srv" "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/trace" @@ -83,21 +85,27 @@ func TestIntegrations(t *testing.T) { check.TestingT(t) } var _ = check.Suite(&IntSuite{}) -func (s *IntSuite) TearDownSuite(c *check.C) { - var err error - // restore os.Stdin to its original condition: connected to /dev/null - os.Stdin.Close() - os.Stdin, err = os.Open("/dev/null") - c.Assert(err, check.IsNil) -} +// TestMain will re-execute Teleport to run a command if "exec" is passed to +// it as an argument. Otherwise it will run tests as normal. +func TestMain(m *testing.M) { + // If the test is re-executing itself, execute the command that comes over + // the pipe. + if len(os.Args) == 2 && os.Args[1] == teleport.ExecSubCommand { + srv.RunCommand() + return + } -func (s *IntSuite) SetUpTest(c *check.C) { - os.RemoveAll(client.FullProfilePath("")) + // Otherwise run tests as normal. + code := m.Run() + os.Exit(code) } func (s *IntSuite) SetUpSuite(c *check.C) { var err error - utils.InitLoggerForTests(testing.Verbose()) + + //utils.InitLoggerForTests(testing.Verbose()) + utils.InitLoggerForTests() + SetTestTimeouts(time.Millisecond * time.Duration(100)) s.priv, s.pub, err = testauthority.New().GenerateKeyPair("") @@ -118,6 +126,18 @@ func (s *IntSuite) SetUpSuite(c *check.C) { } } +func (s *IntSuite) TearDownSuite(c *check.C) { + var err error + // restore os.Stdin to its original condition: connected to /dev/null + os.Stdin.Close() + os.Stdin, err = os.Open("/dev/null") + c.Assert(err, check.IsNil) +} + +func (s *IntSuite) SetUpTest(c *check.C) { + os.RemoveAll(client.FullProfilePath("")) +} + // newTeleport helper returns a created but not started Teleport instance pre-configured // with the current user os.user.Current(). func (s *IntSuite) newUnstartedTeleport(c *check.C, logins []string, enableSSH bool) *TeleInstance { @@ -2731,14 +2751,12 @@ func (s *IntSuite) TestPAM(c *check.C) { inEnabled bool inServiceName string outContains []string - outError bool }{ // 0 - No PAM support, session should work but no PAM related output. { inEnabled: false, inServiceName: "", outContains: []string{}, - outError: false, }, // 1 - PAM enabled, module account and session functions return success. { @@ -2748,21 +2766,18 @@ func (s *IntSuite) TestPAM(c *check.C) { "Account opened successfully.", "Session open successfully.", }, - outError: false, }, // 2 - PAM enabled, module account functions fail. { inEnabled: true, inServiceName: "teleport-acct-failure", outContains: []string{}, - outError: true, }, // 3 - PAM enabled, module session functions fail. { inEnabled: true, inServiceName: "teleport-session-failure", outContains: []string{}, - outError: true, }, } @@ -2804,14 +2819,7 @@ func (s *IntSuite) TestPAM(c *check.C) { termSession.Type("\aecho hi\n\r\aexit\n\r\a") err = cl.SSH(context.TODO(), []string{}, false) - - // If an error is expected (for example PAM does not allow a session to be - // created), this failure needs to be checked here. - if tt.outError { - c.Assert(err, check.NotNil) - } else { - c.Assert(err, check.IsNil) - } + c.Assert(err, check.IsNil) cancel() }() @@ -2824,9 +2832,11 @@ func (s *IntSuite) TestPAM(c *check.C) { } // If any output is expected, check to make sure it was output. - for _, expectedOutput := range tt.outContains { - output := string(termSession.Output(100)) - c.Assert(strings.Contains(output, expectedOutput), check.Equals, true) + if len(tt.outContains) > 0 { + for _, expectedOutput := range tt.outContains { + output := string(termSession.Output(100)) + c.Assert(strings.Contains(output, expectedOutput), check.Equals, true) + } } } } @@ -3778,22 +3788,290 @@ func (s *IntSuite) TestDataTransfer(c *check.C) { c.Assert(eventFields.GetInt(events.DataTransmitted) > KB, check.Equals, true) } -// findEventInLog tries to find an event in the audit log file 10 times. +func (s *IntSuite) TestBPFInteractive(c *check.C) { + tr := utils.NewTracer(utils.ThisFunction()).Start() + defer tr.Stop() + + // Check if BPF tests can be run on this host. + err := canTestBPF() + if err != nil { + c.Skip(fmt.Sprintf("Tests for BPF functionality can not be run: %v.", err)) + return + } + + lsPath, err := exec.LookPath("ls") + c.Assert(err, check.IsNil) + + var tests = []struct { + inSessionRecording string + inBPFEnabled bool + outFound bool + }{ + // For session recorded at the node, enhanced events should be found. + { + inSessionRecording: services.RecordAtNode, + inBPFEnabled: true, + outFound: true, + }, + // For session recorded at the node, but BPF is turned off, no events + // should be found. + { + inSessionRecording: services.RecordAtNode, + inBPFEnabled: false, + outFound: false, + }, + // For session recorded at the proxy, enhanced events should not be found. + // BPF turned off simulates an OpenSSH node. + { + inSessionRecording: services.RecordAtProxy, + inBPFEnabled: false, + outFound: false, + }, + } + for _, tt := range tests { + // Create temporary directory where cgroup2 hierarchy will be mounted. + dir, err := ioutil.TempDir("", "cgroup-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir) + + // Create and start a Teleport cluster. + makeConfig := func() (*check.C, []string, []*InstanceSecrets, *service.Config) { + clusterConfig, err := services.NewClusterConfig(services.ClusterConfigSpecV3{ + SessionRecording: tt.inSessionRecording, + LocalAuth: services.NewBool(true), + }) + c.Assert(err, check.IsNil) + + // Create default config. + tconf := service.MakeDefaultConfig() + + // Configure Auth. + tconf.Auth.Preference.SetSecondFactor("off") + tconf.Auth.Enabled = true + tconf.Auth.ClusterConfig = clusterConfig + + // Configure Proxy. + tconf.Proxy.Enabled = true + tconf.Proxy.DisableWebService = false + tconf.Proxy.DisableWebInterface = true + + // Configure Node. If session are being recorded at the proxy, don't enable + // BPF to simulate an OpenSSH node. + tconf.SSH.Enabled = true + if tt.inBPFEnabled { + tconf.SSH.BPF.Enabled = true + tconf.SSH.BPF.CgroupPath = dir + } + return c, nil, nil, tconf + } + main := s.newTeleportWithConfig(makeConfig()) + defer main.Stop(true) + + // Create a client terminal and context to signal when the client is done + // with the terminal. + term := NewTerminal(250) + doneContext, doneCancel := context.WithCancel(context.Background()) + + func() { + client, err := main.NewClient(ClientConfig{ + Login: s.me.Username, + Cluster: Site, + Host: Host, + Port: main.GetPortSSHInt(), + }) + c.Assert(err, check.IsNil) + + // Connect terminal to std{in,out} of client. + client.Stdout = &term + client.Stdin = &term + + // "Type" a command into the terminal. + term.Type(fmt.Sprintf("\a%v\n\r\aexit\n\r\a", lsPath)) + err = client.SSH(context.TODO(), []string{}, false) + c.Assert(err, check.IsNil) + + // Signal that the client has finished the interactive session. + doneCancel() + }() + + // Wait 10 seconds for the client to finish up the interactive session. + select { + case <-time.After(10 * time.Second): + c.Fatalf("Timed out waiting for client to finish interactive session.") + case <-doneContext.Done(): + } + + // Enhanced events should show up for session recorded at the node but not + // at the proxy. + if tt.outFound { + _, err = findCommandEventInLog(main, events.SessionCommandEvent, lsPath) + c.Assert(err, check.IsNil) + } else { + _, err = findCommandEventInLog(main, events.SessionCommandEvent, lsPath) + c.Assert(err, check.NotNil) + } + } +} + +func (s *IntSuite) TestBPFExec(c *check.C) { + tr := utils.NewTracer(utils.ThisFunction()).Start() + defer tr.Stop() + + // Check if BPF tests can be run on this host. + err := canTestBPF() + if err != nil { + c.Skip(fmt.Sprintf("Tests for BPF functionality can not be run: %v.", err)) + return + } + + lsPath, err := exec.LookPath("ls") + c.Assert(err, check.IsNil) + + var tests = []struct { + inSessionRecording string + inBPFEnabled bool + outFound bool + }{ + // For session recorded at the node, enhanced events should be found. + { + inSessionRecording: services.RecordAtNode, + inBPFEnabled: true, + outFound: true, + }, + // For session recorded at the node, but BPF is turned off, no events + // should be found. + { + inSessionRecording: services.RecordAtNode, + inBPFEnabled: false, + outFound: false, + }, + // For session recorded at the proxy, enhanced events should not be found. + // BPF turned off simulates an OpenSSH node. + { + inSessionRecording: services.RecordAtProxy, + inBPFEnabled: false, + outFound: false, + }, + } + for _, tt := range tests { + // Create temporary directory where cgroup2 hierarchy will be mounted. + dir, err := ioutil.TempDir("", "cgroup-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir) + + // Create and start a Teleport cluster. + makeConfig := func() (*check.C, []string, []*InstanceSecrets, *service.Config) { + clusterConfig, err := services.NewClusterConfig(services.ClusterConfigSpecV3{ + SessionRecording: tt.inSessionRecording, + LocalAuth: services.NewBool(true), + }) + c.Assert(err, check.IsNil) + + // Create default config. + tconf := service.MakeDefaultConfig() + + // Configure Auth. + tconf.Auth.Preference.SetSecondFactor("off") + tconf.Auth.Enabled = true + tconf.Auth.ClusterConfig = clusterConfig + + // Configure Proxy. + tconf.Proxy.Enabled = true + tconf.Proxy.DisableWebService = false + tconf.Proxy.DisableWebInterface = true + + // Configure Node. If session are being recorded at the proxy, don't enable + // BPF to simulate an OpenSSH node. + tconf.SSH.Enabled = true + if tt.inBPFEnabled { + tconf.SSH.BPF.Enabled = true + tconf.SSH.BPF.CgroupPath = dir + } + return c, nil, nil, tconf + } + main := s.newTeleportWithConfig(makeConfig()) + defer main.Stop(true) + + // Create a client to the above Teleport cluster. + clientConfig := ClientConfig{ + Login: s.me.Username, + Cluster: Site, + Host: Host, + Port: main.GetPortSSHInt(), + } + + // Run exec command. + _, err = runCommand(main, []string{lsPath}, clientConfig, 1) + c.Assert(err, check.IsNil) + + // Enhanced events should show up for session recorded at the node but not + // at the proxy. + if tt.outFound { + _, err = findCommandEventInLog(main, events.SessionCommandEvent, lsPath) + c.Assert(err, check.IsNil) + } else { + _, err = findCommandEventInLog(main, events.SessionCommandEvent, lsPath) + c.Assert(err, check.NotNil) + } + } +} + +// findEventInLog polls the event log looking for an event of a particular type. func findEventInLog(t *TeleInstance, eventName string) (events.EventFields, error) { for i := 0; i < 10; i++ { - eventFields, err := eventInLog(t.Config.DataDir+"/log/events.log", eventName) + eventFields, err := eventsInLog(t.Config.DataDir+"/log/events.log", eventName) if err != nil { time.Sleep(1 * time.Second) continue } - return eventFields, nil + for _, fields := range eventFields { + eventType, ok := fields[events.EventType] + if !ok { + return nil, trace.BadParameter("not found") + } + if eventType == eventName { + return fields, nil + } + } + + time.Sleep(250 * time.Millisecond) } return nil, trace.NotFound("event not found") } -// eventInLog finds event in audit log file. -func eventInLog(path string, eventName string) (events.EventFields, error) { +// findCommandEventInLog polls the event log looking for an event of a particular type. +func findCommandEventInLog(t *TeleInstance, eventName string, programName string) (events.EventFields, error) { + for i := 0; i < 10; i++ { + eventFields, err := eventsInLog(t.Config.DataDir+"/log/events.log", eventName) + if err != nil { + time.Sleep(1 * time.Second) + continue + } + + for _, fields := range eventFields { + eventType, ok := fields[events.EventType] + if !ok { + continue + } + eventPath, ok := fields[events.Path] + if !ok { + continue + } + if eventType == eventName && eventPath == programName { + return fields, nil + } + } + + time.Sleep(1 * time.Second) + } + return nil, trace.NotFound("event not found") +} + +// eventsInLog returns all events in a log file. +func eventsInLog(path string, eventName string) ([]events.EventFields, error) { + var ret []events.EventFields + file, err := os.Open(path) if err != nil { return nil, trace.Wrap(err) @@ -3807,17 +4085,13 @@ func eventInLog(path string, eventName string) (events.EventFields, error) { if err != nil { return nil, trace.Wrap(err) } - - eventType, ok := fields[events.EventType] - if !ok { - return nil, trace.BadParameter("not found") - } - if eventType == eventName { - return fields, nil - } + ret = append(ret, fields) } - return nil, trace.NotFound("event not found") + if len(ret) == 0 { + return nil, trace.NotFound("event not found") + } + return ret, nil } // runCommand is a shortcut for running SSH command, it creates a client @@ -3934,3 +4208,18 @@ func hasPAMPolicy() bool { return true } + +// isRoot returns a boolean if the test is being run as root or not. Tests +// for this package must be run as root. +func canTestBPF() error { + if os.Geteuid() != 0 { + return trace.BadParameter("not root") + } + + err := bpf.IsHostCompatible() + if err != nil { + return trace.Wrap(err) + } + + return nil +} diff --git a/lib/auth/clt.go b/lib/auth/clt.go index 962f155cd29..5bb6fe95216 100644 --- a/lib/auth/clt.go +++ b/lib/auth/clt.go @@ -241,6 +241,15 @@ func NewTLSClient(cfg ClientConfig, params ...roundtrip.ClientParam) (*Client, e MaxIdleConns: defaults.HTTPMaxIdleConns, MaxIdleConnsPerHost: defaults.HTTPMaxIdleConnsPerHost, + // Limit the total number of connections to the Auth Server. Some hosts allow a low + // number of connections per process (ulimit) to a host. This is a problem for + // enhanced session recording auditing which emits so many events to the + // Audit Log (using the Auth Client) that the connection pool often does not + // have a free connection to return, so just opens a new one. This quickly + // leads to hitting the OS limit and the client returning out of file + // descriptors error. + MaxConnsPerHost: defaults.HTTPMaxConnsPerHost, + // IdleConnTimeout defines the maximum amount of time before idle connections // are closed. Leaving this unset will lead to connections open forever and // will cause memory leaks in a long running process. diff --git a/lib/auth/init.go b/lib/auth/init.go index 45370426080..0340c174f20 100644 --- a/lib/auth/init.go +++ b/lib/auth/init.go @@ -437,6 +437,12 @@ func migrateLegacyResources(cfg InitConfig, asrv *AuthServer) error { if err != nil { return trace.Wrap(err) } + + err = migrateRoleOptions(asrv) + if err != nil { + return trace.Wrap(err) + } + return nil } @@ -922,3 +928,30 @@ func migrateRemoteClusters(asrv *AuthServer) error { return nil } + +// DELETE IN: 4.3.0. +// migrateRoleOptions adds the "enhanced_recording" option to all roles. +func migrateRoleOptions(asrv *AuthServer) error { + roles, err := asrv.GetRoles() + if err != nil { + return trace.Wrap(err) + } + + for _, role := range roles { + options := role.GetOptions() + if options.BPF == nil { + fmt.Printf("--> Migrating role %v. Added default enhanced events.", role.GetName()) + log.Debugf("Migrating role %v. Added default enhanced events.", role.GetName()) + options.BPF = defaults.EnhancedEvents() + } else { + continue + } + role.SetOptions(options) + err := asrv.UpsertRole(role) + if err != nil { + return trace.Wrap(err) + } + } + + return nil +} diff --git a/lib/auth/init_test.go b/lib/auth/init_test.go index 6bf0957f0fb..428974b9696 100644 --- a/lib/auth/init_test.go +++ b/lib/auth/init_test.go @@ -315,3 +315,8 @@ func (s *AuthInitSuite) TestClusterName(c *C) { c.Assert(err, IsNil) c.Assert(cn.GetClusterName(), Equals, "me.localhost") } + +// DELETE IN: 4.3.0 +func (s *AuthInitSuite) TestRoleOptions(c *C) { + // TODO: Implement. +} diff --git a/lib/auth/tls_test.go b/lib/auth/tls_test.go index 7460d26da09..81666f263a8 100644 --- a/lib/auth/tls_test.go +++ b/lib/auth/tls_test.go @@ -1328,8 +1328,7 @@ func (s *TLSSuite) TestGetCertAuthority(c *check.C) { user, err := services.NewUser("bob") c.Assert(err, check.IsNil) - role := services.NewImplicitRole() - role.SetName(user.GetName()) + role := services.RoleForUser(user) role.SetLogins(services.Allow, []string{user.GetName()}) err = s.server.Auth().UpsertRole(role) c.Assert(err, check.IsNil) diff --git a/lib/bpf/bpf.go b/lib/bpf/bpf.go new file mode 100644 index 00000000000..030972babd8 --- /dev/null +++ b/lib/bpf/bpf.go @@ -0,0 +1,480 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +// #cgo LDFLAGS: -ldl +// #include +import "C" + +import ( + "bytes" + "context" + "encoding/binary" + "net" + "strconv" + "sync" + "time" + "unsafe" + + "github.com/gravitational/teleport" + controlgroup "github.com/gravitational/teleport/lib/cgroup" + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/events" + + "github.com/gravitational/trace" + "github.com/gravitational/ttlmap" + + "github.com/iovisor/gobpf/bcc" +) + +// Service manages BPF and control groups orchestration. +type Service struct { + *Config + + // watch is a map of cgroup IDs that the BPF service is watching and + // emitting events for. + watch map[uint64]*SessionContext + watchMu sync.Mutex + + // argsCache holds the arguments to execve because they come a different + // event than the result. + argsCache *ttlmap.TTLMap + + // closeContext is used to signal the BPF service is shutting down to all + // goroutines. + closeContext context.Context + closeFunc context.CancelFunc + + // cgroup is used to manage control groups. + cgroup *controlgroup.Service + + // exec holds a BPF program that hooks execve. + exec *exec + + // open holds a BPF program that hooks openat. + open *open + + // conn is a BPF programs that hooks connect. + conn *conn +} + +// New creates a BPF service. +func New(config *Config) (BPF, error) { + err := config.CheckAndSetDefaults() + if err != nil { + return nil, trace.Wrap(err) + } + + // If BPF-based auditing is not enabled, don't configure anything return + // right away. + if !config.Enabled { + return &NOP{}, nil + } + + // Check if the host can run BPF programs. + err = IsHostCompatible() + if err != nil { + return nil, trace.Wrap(err) + } + + // Create a cgroup controller to add/remote cgroups. + cgroup, err := controlgroup.New(&controlgroup.Config{ + MountPath: config.CgroupPath, + }) + if err != nil { + return nil, trace.Wrap(err) + } + + closeContext, closeFunc := context.WithCancel(context.Background()) + + s := &Service{ + Config: config, + + watch: make(map[uint64]*SessionContext), + + closeContext: closeContext, + closeFunc: closeFunc, + + cgroup: cgroup, + } + + // Create args cache used by the exec BPF program. + s.argsCache, err = ttlmap.New(defaults.ArgsCacheSize) + if err != nil { + return nil, trace.Wrap(err) + } + + // Compile and start BPF programs. + s.exec, err = startExec(closeContext, config.CommandBufferSize) + if err != nil { + return nil, trace.Wrap(err) + } + s.open, err = startOpen(closeContext, config.DiskBufferSize) + if err != nil { + return nil, trace.Wrap(err) + } + s.conn, err = startConn(closeContext, config.NetworkBufferSize) + if err != nil { + return nil, trace.Wrap(err) + } + + log.Debugf("Started enhanced auditing with buffer sizes %v %v %v and cgroup mount path: %v.", + s.CommandBufferSize, s.DiskBufferSize, s.NetworkBufferSize, s.CgroupPath) + + // Start pulling events off the perf buffers and emitting them to the + // Audit Log. + go s.loop() + + return s, nil +} + +// Close will stop any running BPF programs. Note this is only for a graceful +// shutdown, from the man page for BPF: "Generally, eBPF programs are loaded +// by the user process and automatically unloaded when the process exits." +func (s *Service) Close() error { + // Unload the BPF programs. + s.exec.close() + s.open.close() + s.conn.close() + + // Close cgroup service. + s.cgroup.Close() + + // Signal to the loop pulling events off the perf buffer to shutdown. + s.closeFunc() + + return nil +} + +// OpenSession will place a process within a cgroup and being monitoring all +// events from that cgroup and emitting the results to the audit log. +func (s *Service) OpenSession(ctx *SessionContext) (uint64, error) { + err := s.cgroup.Create(ctx.SessionID) + if err != nil { + return 0, trace.Wrap(err) + } + + cgroupID, err := s.cgroup.ID(ctx.SessionID) + if err != nil { + return 0, trace.Wrap(err) + } + + // Start watching for any events that come from this cgroup. + s.addWatch(cgroupID, ctx) + + // Place requested PID into cgroup. + err = s.cgroup.Place(ctx.SessionID, ctx.PID) + if err != nil { + return 0, trace.Wrap(err) + } + + return cgroupID, nil +} + +// CloseSession will stop monitoring events from a particular cgroup and +// remove the cgroup. +func (s *Service) CloseSession(ctx *SessionContext) error { + cgroupID, err := s.cgroup.ID(ctx.SessionID) + if err != nil { + return trace.Wrap(err) + } + + // Stop watching for events from this PID. + s.removeWatch(cgroupID) + + // Move all PIDs to the root cgroup and remove the cgroup created for this + // session. + err = s.cgroup.Remove(ctx.SessionID) + if err != nil { + return trace.Wrap(err) + } + + return nil +} + +// loop pulls events off the perf ring buffer, parses them, and emits them to +// the audit log. +func (s *Service) loop() { + for { + select { + // Program execution. + case event := <-s.exec.events(): + s.emitCommandEvent(event) + // Disk access. + case event := <-s.open.events(): + s.emitDiskEvent(event) + // Network access (IPv4). + case event := <-s.conn.v4Events(): + s.emit4NetworkEvent(event) + // Network access (IPv4). + case event := <-s.conn.v6Events(): + s.emit6NetworkEvent(event) + case <-s.closeContext.Done(): + return + } + } +} + +// emitCommandEvent will parse and emit command events to the Audit Log. +func (s *Service) emitCommandEvent(eventBytes []byte) { + // Unmarshal raw event bytes. + var event rawExecEvent + err := unmarshalEvent(eventBytes, &event) + if err != nil { + log.Debugf("Failed to read binary data: %v.", err) + return + } + + // If the event comes from a unmonitored process/cgroup, don't process it. + ctx, ok := s.watch[event.CgroupID] + if !ok { + return + } + + // If the command event is not being monitored, don't process it. + _, ok = ctx.Events[teleport.EnhancedRecordingCommand] + if !ok { + return + } + + switch event.Type { + // Args are sent in their own event by execsnoop to save stack space. Store + // the args in a ttlmap so they can be retrieved when the return event arrives. + case eventArg: + var buf []string + buffer, ok := s.argsCache.Get(strconv.FormatUint(event.PID, 10)) + if !ok { + buf = make([]string, 0) + } else { + buf = buffer.([]string) + } + + argv := (*C.char)(unsafe.Pointer(&event.Argv)) + buf = append(buf, C.GoString(argv)) + s.argsCache.Set(strconv.FormatUint(event.PID, 10), buf, 24*time.Hour) + // The event has returned, emit the fully parsed event. + case eventRet: + // The args should have come in a previous event, find them by PID. + args, ok := s.argsCache.Get(strconv.FormatUint(event.PID, 10)) + if !ok { + log.Debugf("Got event with missing args: skipping.") + lostCommandEvents.Add(float64(1)) + return + } + argv := args.([]string) + + // Emit "command" event. + eventFields := events.EventFields{ + // Common fields. + events.EventNamespace: ctx.Namespace, + events.SessionEventID: ctx.SessionID, + events.SessionServerID: ctx.ServerID, + events.EventLogin: ctx.Login, + events.EventUser: ctx.User, + // Command fields. + events.PID: event.PPID, + events.PPID: event.PID, + events.CgroupID: event.CgroupID, + events.Program: convertString(unsafe.Pointer(&event.Command)), + events.Path: argv[0], + events.Argv: argv[1:], + events.ReturnCode: event.ReturnCode, + } + ctx.AuditLog.EmitAuditEvent(events.SessionCommand, eventFields) + + // Now that the event has been processed, remove from cache. + s.argsCache.Remove(strconv.FormatUint(event.PID, 10)) + } +} + +// emitDiskEvent will parse and emit disk events to the Audit Log. +func (s *Service) emitDiskEvent(eventBytes []byte) { + // Unmarshal raw event bytes. + var event rawOpenEvent + err := unmarshalEvent(eventBytes, &event) + if err != nil { + log.Debugf("Failed to read binary data: %v.", err) + return + } + + // If the event comes from a unmonitored process/cgroup, don't process it. + ctx, ok := s.watch[event.CgroupID] + if !ok { + return + } + + // If the network event is not being monitored, don't process it. + _, ok = ctx.Events[teleport.EnhancedRecordingDisk] + if !ok { + return + } + + eventFields := events.EventFields{ + // Common fields. + events.EventNamespace: ctx.Namespace, + events.SessionEventID: ctx.SessionID, + events.SessionServerID: ctx.ServerID, + events.EventLogin: ctx.Login, + events.EventUser: ctx.User, + // Disk fields. + events.PID: event.PID, + events.CgroupID: event.CgroupID, + events.Program: convertString(unsafe.Pointer(&event.Command)), + events.Path: convertString(unsafe.Pointer(&event.Path)), + events.Flags: event.Flags, + events.ReturnCode: event.ReturnCode, + } + ctx.AuditLog.EmitAuditEvent(events.SessionDisk, eventFields) +} + +// emit4NetworkEvent will parse and emit IPv4 events to the Audit Log. +func (s *Service) emit4NetworkEvent(eventBytes []byte) { + // Unmarshal raw event bytes. + var event rawConn4Event + err := unmarshalEvent(eventBytes, &event) + if err != nil { + log.Debugf("Failed to read binary data: %v.", err) + return + } + + // If the event comes from a unmonitored process/cgroup, don't process it. + ctx, ok := s.watch[event.CgroupID] + if !ok { + return + } + + // If the network event is not being monitored, don't process it. + _, ok = ctx.Events[teleport.EnhancedRecordingNetwork] + if !ok { + return + } + + // Source. + src := make([]byte, 4) + binary.LittleEndian.PutUint32(src, uint32(event.SrcAddr)) + srcAddr := net.IP(src) + + // Destination. + dst := make([]byte, 4) + binary.LittleEndian.PutUint32(dst, uint32(event.DstAddr)) + dstAddr := net.IP(dst) + + eventFields := events.EventFields{ + // Common fields. + events.EventNamespace: ctx.Namespace, + events.SessionEventID: ctx.SessionID, + events.SessionServerID: ctx.ServerID, + events.EventLogin: ctx.Login, + events.EventUser: ctx.User, + // Network fields. + events.PID: event.PID, + events.CgroupID: event.CgroupID, + events.Program: convertString(unsafe.Pointer(&event.Command)), + events.SrcAddr: srcAddr, + events.DstAddr: dstAddr, + events.DstPort: event.DstPort, + events.TCPVersion: 4, + } + ctx.AuditLog.EmitAuditEvent(events.SessionNetwork, eventFields) +} + +// emit6NetworkEvent will parse and emit IPv6 events to the Audit Log. +func (s *Service) emit6NetworkEvent(eventBytes []byte) { + // Unmarshal raw event bytes. + var event rawConn6Event + err := unmarshalEvent(eventBytes, &event) + if err != nil { + log.Debugf("Failed to read binary data: %v.", err) + return + } + + // If the event comes from a unmonitored process/cgroup, don't process it. + ctx, ok := s.watch[event.CgroupID] + if !ok { + return + } + + // If the network event is not being monitored, don't process it. + _, ok = ctx.Events[teleport.EnhancedRecordingNetwork] + if !ok { + return + } + + // Source. + src := make([]byte, 16) + binary.LittleEndian.PutUint32(src[0:], event.SrcAddr[0]) + binary.LittleEndian.PutUint32(src[4:], event.SrcAddr[1]) + binary.LittleEndian.PutUint32(src[8:], event.SrcAddr[2]) + binary.LittleEndian.PutUint32(src[12:], event.SrcAddr[3]) + srcAddr := net.IP(src) + + // Destination. + dst := make([]byte, 16) + binary.LittleEndian.PutUint32(dst[0:], event.DstAddr[0]) + binary.LittleEndian.PutUint32(dst[4:], event.DstAddr[1]) + binary.LittleEndian.PutUint32(dst[8:], event.DstAddr[2]) + binary.LittleEndian.PutUint32(dst[12:], event.DstAddr[3]) + dstAddr := net.IP(dst) + + eventFields := events.EventFields{ + // Common fields. + events.EventNamespace: ctx.Namespace, + events.SessionEventID: ctx.SessionID, + events.SessionServerID: ctx.ServerID, + events.EventLogin: ctx.Login, + events.EventUser: ctx.User, + // Connect fields. + events.PID: event.PID, + events.CgroupID: event.CgroupID, + events.Program: convertString(unsafe.Pointer(&event.Command)), + events.SrcAddr: srcAddr, + events.DstAddr: dstAddr, + events.DstPort: event.DstPort, + events.TCPVersion: 6, + } + ctx.AuditLog.EmitAuditEvent(events.SessionNetwork, eventFields) +} + +func (s *Service) addWatch(cgroupID uint64, ctx *SessionContext) { + s.watchMu.Lock() + defer s.watchMu.Unlock() + + s.watch[cgroupID] = ctx +} + +func (s *Service) removeWatch(cgroupID uint64) { + s.watchMu.Lock() + defer s.watchMu.Unlock() + + delete(s.watch, cgroupID) +} + +// unmarshalEvent will unmarshal the perf event. +func unmarshalEvent(data []byte, v interface{}) error { + err := binary.Read(bytes.NewBuffer(data), bcc.GetHostByteOrder(), v) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// convertString converts a C string to a Go string. +func convertString(s unsafe.Pointer) string { + return C.GoString((*C.char)(s)) +} diff --git a/lib/bpf/bpf_nop.go b/lib/bpf/bpf_nop.go new file mode 100644 index 00000000000..38697d3a7e8 --- /dev/null +++ b/lib/bpf/bpf_nop.go @@ -0,0 +1,30 @@ +// +build !bpf 386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +// Service is used on non-Linux systems as a NOP service that allows the +// caller to open and close sessions that do nothing on systems that don't +// support eBPF. +type Service struct { +} + +// New returns a new NOP service. Note this function does nothing. +func New(config *Config) (BPF, error) { + return &NOP{}, nil +} diff --git a/lib/bpf/bpf_test.go b/lib/bpf/bpf_test.go new file mode 100644 index 00000000000..c0979b275a0 --- /dev/null +++ b/lib/bpf/bpf_test.go @@ -0,0 +1,509 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "os" + os_exec "os/exec" + "sync" + "testing" + "time" + "unsafe" + + "github.com/gravitational/trace" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/events" + "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/utils" + + "github.com/pborman/uuid" + "gopkg.in/check.v1" +) + +type Suite struct{} + +var _ = fmt.Printf +var _ = check.Suite(&Suite{}) + +func TestBPF(t *testing.T) { check.TestingT(t) } + +func (s *Suite) SetUpSuite(c *check.C) { + utils.InitLoggerForTests() +} +func (s *Suite) TearDownSuite(c *check.C) {} +func (s *Suite) SetUpTest(c *check.C) {} +func (s *Suite) TearDownTest(c *check.C) {} + +func (s *Suite) TestWatch(c *check.C) { + // This test must be run as root and the host has to be capable of running + // BPF programs. + if !isRoot() { + c.Skip("Tests for package bpf can only be run as root.") + } + err := IsHostCompatible() + if err != nil { + c.Skip(fmt.Sprintf("Tests for package bpf can not be run: %v.", err)) + } + + // Create temporary directory where cgroup2 hierarchy will be mounted. + dir, err := ioutil.TempDir("", "cgroup-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir) + + // Create BPF service. + service, err := New(&Config{ + Enabled: true, + CgroupPath: dir, + }) + + // Create a fake audit log that can be used to capture the events emitted. + auditLog := newFakeLog() + + // Create and start a program that does nothing. Since sleep will run longer + // than we wait below, nothing should be emit to the Audit Log. + cmd := os_exec.Command("sleep", "10") + err = cmd.Start() + c.Assert(err, check.IsNil) + + // Create a monitoring session for init. The events we execute should not + // have PID 1, so nothing should be captured in the Audit Log. + cgroupID, err := service.OpenSession(&SessionContext{ + Namespace: defaults.Namespace, + SessionID: uuid.New(), + ServerID: uuid.New(), + Login: "foo", + User: "foo@example.com", + PID: cmd.Process.Pid, + AuditLog: auditLog, + Events: map[string]bool{ + teleport.EnhancedRecordingCommand: true, + teleport.EnhancedRecordingDisk: true, + teleport.EnhancedRecordingNetwork: true, + }, + }) + c.Assert(err, check.IsNil) + c.Assert(cgroupID > 0, check.Equals, true) + + // Execute "ls" in a loop. + go func() { + for { + // Find "ls" binary. + lsPath, err := os_exec.LookPath("ls") + c.Assert(err, check.IsNil) + + // Run "ls". + err = os_exec.Command(lsPath).Run() + c.Assert(err, check.IsNil) + + // Delay. + time.Sleep(250 * time.Millisecond) + } + }() + + // Keep checking that even though events are being executed, that they are + // not emitted to the audit log because the cgroup they are in is not being + // monitored. + timer := time.NewTimer(250 * time.Millisecond) + defer timer.Stop() + for { + select { + case <-time.Tick(250 * time.Millisecond): + c.Assert(auditLog.events, check.HasLen, 0) + case <-timer.C: + return + } + } +} + +// TestObfuscate checks if execsnoop can capture Obfuscated commands. +func (s *Suite) TestObfuscate(c *check.C) { + // This test must be run as root and the host has to be capable of running + // BPF programs. + if !isRoot() { + c.Skip("Tests for package bpf can only be run as root.") + return + } + err := IsHostCompatible() + if err != nil { + c.Skip(fmt.Sprintf("Tests for package bpf can not be run: %v.", err)) + return + } + + // Find the programs needed to run these tests on the host. + decoderPath, err := os_exec.LookPath("base64") + c.Assert(err, check.IsNil) + shellPath, err := os_exec.LookPath("sh") + c.Assert(err, check.IsNil) + + // Create a context that will be used to close and stop the BPF programs + // at the end of the test. + closeContext, closeFunc := context.WithCancel(context.Background()) + defer closeFunc() + + // Start execsnoop. + execsnoop, err := startExec(closeContext, defaults.PerfBufferPageCount) + defer execsnoop.close() + c.Assert(err, check.IsNil) + + // Create a context that will be used to signal that an event has been recieved. + doneContext, doneFunc := context.WithCancel(context.Background()) + + // Start two goroutines. The first writes a script which will execute "ls" + // in a loop. The second waits for an exec event to show up the reports "ls" + // has been executed. + go func() { + // Create temporary file. + file, err := ioutil.TempFile("", "test-script") + c.Assert(err, check.IsNil) + defer os.Remove(file.Name()) + + // Write script to file. + shellContents := fmt.Sprintf("#!%v\necho bHM= | %v --decode | %v", + shellPath, decoderPath, shellPath) + _, err = file.Write([]byte(shellContents)) + c.Assert(err, check.IsNil) + err = file.Close() + c.Assert(err, check.IsNil) + + // Make script executable. + err = os.Chmod(file.Name(), 0700) + c.Assert(err, check.IsNil) + + for { + // Run script. + err = os_exec.Command(file.Name()).Run() + c.Assert(err, check.IsNil) + + // Delay. + time.Sleep(250 * time.Millisecond) + } + }() + go func() { + for { + select { + case eventBytes := <-execsnoop.events(): + // Unmarshal the event. + var event rawExecEvent + err := unmarshalEvent(eventBytes, &event) + c.Assert(err, check.IsNil) + + // Check the event is what we expect, in this case "ls". + if convertString(unsafe.Pointer(&event.Command)) == "ls" { + doneFunc() + } + } + } + + }() + + // Wait for an event to arrive from execsnoop. If an event does not arrive + // within 10 seconds, timeout. + select { + case <-doneContext.Done(): + case <-time.After(10 * time.Second): + c.Fatalf("Timed out waiting for an event.") + } + +} + +// TestScript checks if execsnoop can capture what a script executes. +func (s *Suite) TestScript(c *check.C) { + // This test must be run as root and the host has to be capable of running + // BPF programs. + if !isRoot() { + c.Skip("Tests for package bpf can only be run as root.") + } + err := IsHostCompatible() + if err != nil { + c.Skip(fmt.Sprintf("Tests for package bpf can not be run: %v.", err)) + } + + // Create a context that will be used to close and stop the BPF programs + // at the end of the test. + closeContext, closeFunc := context.WithCancel(context.Background()) + defer closeFunc() + + // Start execsnoop. + execsnoop, err := startExec(closeContext, defaults.PerfBufferPageCount) + defer execsnoop.close() + c.Assert(err, check.IsNil) + + // Create a context that will be used to signal that an event has been recieved. + doneContext, doneFunc := context.WithCancel(context.Background()) + + // Start two goroutines. The first writes a script which will execute "ls" + // in a loop. The second waits for an exec event to show up the reports "ls" + // has been executed. + go func() { + // Create temporary file. + file, err := ioutil.TempFile("", "test-script") + c.Assert(err, check.IsNil) + defer os.Remove(file.Name()) + + // Write script to file. + _, err = file.Write([]byte("#!/bin/sh\nls")) + c.Assert(err, check.IsNil) + err = file.Close() + c.Assert(err, check.IsNil) + + // Make script executable. + err = os.Chmod(file.Name(), 0700) + c.Assert(err, check.IsNil) + + for { + // Run script. + err = os_exec.Command(file.Name()).Run() + c.Assert(err, check.IsNil) + // Delay. + time.Sleep(250 * time.Millisecond) + } + }() + go func() { + for { + select { + case eventBytes := <-execsnoop.events(): + // Unmarshal the event. + var event rawExecEvent + err := unmarshalEvent(eventBytes, &event) + c.Assert(err, check.IsNil) + + // Check the event is what we expect, in this case "ls". + if convertString(unsafe.Pointer(&event.Command)) == "ls" { + doneFunc() + } + } + } + + }() + + // Wait for an event to arrive from execsnoop. If an event does not arrive + // within 10 seconds, timeout. + select { + case <-doneContext.Done(): + case <-time.After(10 * time.Second): + c.Fatalf("Timed out waiting for an event.") + } +} + +// TestPrograms tests execsnoop, opensnoop, and tcpconnect to make sure they +// run and receive events. +func (s *Suite) TestPrograms(c *check.C) { + // This test must be run as root. Only root can create cgroups. + if !isRoot() { + c.Skip("Tests for package bpf can only be run as root.") + } + + // Check that the host is capable of running BPF programs. + err := IsHostCompatible() + if err != nil { + c.Skip(fmt.Sprintf("Tests for package bpf can not be run: %v.", err)) + } + + // Start a debug server that tcpconnect will connect to. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "hello, world") + })) + defer ts.Close() + + // Create a context that will be used to close and stop the BPF programs + // at the end of the test. + closeContext, closeFunc := context.WithCancel(context.Background()) + defer closeFunc() + + // Start execsnoop. + execsnoop, err := startExec(closeContext, defaults.PerfBufferPageCount) + defer execsnoop.close() + c.Assert(err, check.IsNil) + + // Start opensnoop. + opensnoop, err := startOpen(closeContext, defaults.PerfBufferPageCount) + defer opensnoop.close() + c.Assert(err, check.IsNil) + + // Start tcpconnect. + tcpconnect, err := startConn(closeContext, defaults.PerfBufferPageCount) + defer tcpconnect.close() + c.Assert(err, check.IsNil) + + // Loop over all three programs and make sure events are received off the + // perf buffer. + var tests = []struct { + inName string + inCommand string + inCommandArgs []string + inEventCh <-chan []byte + inHTTP bool + }{ + // Run execsnoop with "ls". + { + inName: "execsnoop", + inCommand: "ls", + inCommandArgs: []string{}, + inEventCh: execsnoop.events(), + inHTTP: false, + }, + // Run opensnoop with "ls". This is fine because "ls" will open some + // shared library. + { + inName: "opensnoop", + inCommand: "ls", + inCommandArgs: []string{}, + inEventCh: opensnoop.events(), + inHTTP: false, + }, + // Run tcpconnect with netcat. + { + inName: "tcpconnect", + inEventCh: tcpconnect.v4Events(), + inHTTP: true, + }, + } + for _, tt := range tests { + // Create a context that will be used to signal that an event has been recieved. + doneContext, doneFunc := context.WithCancel(context.Background()) + + // Start two goroutines. The first will wait for the BPF program event to + // arrive, and once it has, signal over the context that it's complete. The + // second will continue to execute or a HTTP GET in a in a loop attempting to + // trigger an event. + go waitForEvent(doneContext, doneFunc, tt.inEventCh) + if tt.inHTTP { + go executeHTTP(c, doneContext, ts.URL) + } else { + go executeCommand(c, doneContext, tt.inCommand) + } + + // Wait for an event to arrive from execsnoop. If an event does not arrive + // within 10 seconds, timeout. + select { + case <-doneContext.Done(): + case <-time.After(10 * time.Second): + c.Fatalf("Timed out waiting for an %v event.", tt.inName) + } + } +} + +// waitForEvent will wait for an event to arrive over the perf buffer and +// signal when it has. +func waitForEvent(ctx context.Context, cancel context.CancelFunc, eventCh <-chan []byte) { + for { + select { + case <-eventCh: + cancel() + case <-ctx.Done(): + return + } + } +} + +// executeCommand will execute some command in a loop. +func executeCommand(c *check.C, doneContext context.Context, file string) { + for { + // Lookup and run the requested command. + path, err := os_exec.LookPath(file) + if err != nil { + c.Fatalf("Failed to find execute %q: %v.", file, err) + } + err = os_exec.Command(path).Run() + if err != nil { + c.Fatalf("Failed to run command %q: %v.", file, err) + } + + time.Sleep(250 * time.Millisecond) + } +} + +// executeHTTP will perform a HTTP GET to some endpoint in a loop. +func executeHTTP(c *check.C, doneContext context.Context, endpoint string) { + for { + // Perform HTTP GET to the requested endpoint. + _, err := http.Get(endpoint) + c.Assert(err, check.IsNil) + + time.Sleep(250 * time.Millisecond) + } +} + +// fakeLog is used in tests to obtain events emitted to the Audit Log. +type fakeLog struct { + mu sync.Mutex + events []events.EventFields +} + +func newFakeLog() *fakeLog { + return &fakeLog{ + events: make([]events.EventFields, 0), + } +} + +func (a *fakeLog) EmitAuditEvent(e events.Event, f events.EventFields) error { + a.mu.Lock() + defer a.mu.Unlock() + + a.events = append(a.events, f) + return nil +} + +func (a *fakeLog) PostSessionSlice(s events.SessionSlice) error { + return trace.NotImplemented("not implemented") +} + +func (a *fakeLog) UploadSessionRecording(r events.SessionRecording) error { + return trace.NotImplemented("not implemented") +} + +func (a *fakeLog) GetSessionChunk(namespace string, sid session.ID, offsetBytes int, maxBytes int) ([]byte, error) { + return nil, trace.NotFound("") +} + +func (a *fakeLog) GetSessionEvents(namespace string, sid session.ID, after int, includePrintEvents bool) ([]events.EventFields, error) { + return nil, trace.NotFound("") +} + +func (a *fakeLog) SearchEvents(fromUTC, toUTC time.Time, query string, limit int) ([]events.EventFields, error) { + return nil, trace.NotFound("") +} + +func (a *fakeLog) SearchSessionEvents(fromUTC time.Time, toUTC time.Time, limit int) ([]events.EventFields, error) { + return nil, trace.NotFound("") +} + +func (a *fakeLog) WaitForDelivery(context.Context) error { + return trace.NotImplemented("not implemented") +} + +func (a *fakeLog) Close() error { + return trace.NotFound("") +} + +// isRoot returns a boolean if the test is being run as root or not. Tests +// for this package must be run as root. +func isRoot() bool { + if os.Geteuid() != 0 { + return false + } + return true +} diff --git a/lib/bpf/command.go b/lib/bpf/command.go new file mode 100644 index 00000000000..0f096d89e85 --- /dev/null +++ b/lib/bpf/command.go @@ -0,0 +1,249 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +import "C" + +import ( + "context" + + "github.com/gravitational/teleport" + + "github.com/gravitational/trace" + + "github.com/iovisor/gobpf/bcc" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + lostCommandEvents = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: teleport.MetricLostCommandEvents, + Help: "Number of lost command events.", + }, + ) +) + +func init() { + prometheus.MustRegister(lostCommandEvents) +} + +// rawExecEvent is sent by the eBPF program that Teleport pulls off the perf +// buffer. +type rawExecEvent struct { + // PID is the ID of the process. + PID uint64 + + // PPID is the PID of the parent process. + PPID uint64 + + // Command is the executable. + Command [commMax]byte + + // Type is the type of event. + Type int32 + + // Argv is the list of arguments to the program. + Argv [argvMax]byte + + // ReturnCode is the return code of execve. + ReturnCode int32 + + // CgroupID is the internal cgroupv2 ID of the event. + CgroupID uint64 +} + +// exec runs a BPF program (execsnoop) that hooks execve. +type exec struct { + closeContext context.Context + + eventCh <-chan []byte + lostCh <-chan uint64 + + perfMaps []*bcc.PerfMap + module *bcc.Module +} + +// startExec will compile, load, start, and pull events off the perf buffer +// for the BPF program. +func startExec(closeContext context.Context, pageCount int) (*exec, error) { + var err error + + e := &exec{ + closeContext: closeContext, + } + + // Compile the BPF program. + e.module = bcc.NewModule(execveSource, []string{}) + if e.module == nil { + return nil, trace.BadParameter("failed to load libbcc") + } + + // Hook execve syscall. + err = attachProbe(e.module, bcc.GetSyscallFnName("execve"), "syscall__execve") + if err != nil { + return nil, trace.Wrap(err) + } + err = attachRetProbe(e.module, bcc.GetSyscallFnName("execve"), "do_ret_sys_execve") + if err != nil { + return nil, trace.Wrap(err) + } + + // Open perf buffer and start processing execve events. + e.eventCh, e.lostCh, err = openPerfBuffer(e.module, e.perfMaps, pageCount, "execve_events") + if err != nil { + return nil, trace.Wrap(err) + } + + // Start a loop that will emit lost events to prometheus. + go e.lostLoop() + + return e, nil +} + +// close will stop reading events off the perf buffer and unload the BPF +// program. +func (e *exec) close() { + for _, perfMap := range e.perfMaps { + perfMap.Stop() + } + e.module.Close() +} + +// events contains raw events off the perf buffer. +func (e *exec) events() <-chan []byte { + return e.eventCh +} + +// lostLoop keeps emitting the number of lost events to prometheus. +func (e *exec) lostLoop() { + for { + select { + case n := <-e.lostCh: + log.Debugf("Lost %v command events.", n) + lostCommandEvents.Add(float64(n)) + case <-e.closeContext.Done(): + return + } + } +} + +const execveSource string = ` +#include +#include +#include + +#define ARGSIZE 128 + +enum event_type { + EVENT_ARG, + EVENT_RET, +}; + +struct data_t { + // pid as in the userspace term (i.e. task->tgid in kernel). + u64 pid; + // ppid is the userspace term (i.e task->real_parent->tgid in kernel). + u64 ppid; + char comm[TASK_COMM_LEN]; + enum event_type type; + char argv[ARGSIZE]; + int retval; + u64 cgroup; +}; + +BPF_PERF_OUTPUT(execve_events); + +static int __submit_arg(struct pt_regs *ctx, void *ptr, struct data_t *data) +{ + bpf_probe_read(data->argv, sizeof(data->argv), ptr); + execve_events.perf_submit(ctx, data, sizeof(struct data_t)); + return 1; +} + +static int submit_arg(struct pt_regs *ctx, void *ptr, struct data_t *data) +{ + const char *argp = NULL; + bpf_probe_read(&argp, sizeof(argp), ptr); + if (argp) { + return __submit_arg(ctx, (void *)(argp), data); + } + return 0; +} + +int syscall__execve(struct pt_regs *ctx, + const char __user *filename, + const char __user *const __user *__argv, + const char __user *const __user *__envp) +{ + // create data here and pass to submit_arg to save stack space (#555) + struct data_t data = {}; + struct task_struct *task; + + data.pid = bpf_get_current_pid_tgid() >> 32; + data.cgroup = bpf_get_current_cgroup_id(); + + task = (struct task_struct *)bpf_get_current_task(); + // Some kernels, like Ubuntu 4.13.0-generic, return 0 + // as the real_parent->tgid. + // We use the getPpid function as a fallback in those cases. + // See https://github.com/iovisor/bcc/issues/1883. + data.ppid = task->real_parent->tgid; + + bpf_get_current_comm(&data.comm, sizeof(data.comm)); + data.type = EVENT_ARG; + + __submit_arg(ctx, (void *)filename, &data); + + // skip first arg, as we submitted filename + #pragma unroll + for (int i = 1; i < 20; i++) { + if (submit_arg(ctx, (void *)&__argv[i], &data) == 0) + goto out; + } + + // handle truncated argument list + char ellipsis[] = "..."; + __submit_arg(ctx, (void *)ellipsis, &data); +out: + return 0; +} + +int do_ret_sys_execve(struct pt_regs *ctx) +{ + struct data_t data = {}; + struct task_struct *task; + + data.pid = bpf_get_current_pid_tgid() >> 32; + data.cgroup = bpf_get_current_cgroup_id(); + + task = (struct task_struct *)bpf_get_current_task(); + // Some kernels, like Ubuntu 4.13.0-generic, return 0 + // as the real_parent->tgid. + // We use the getPpid function as a fallback in those cases. + // See https://github.com/iovisor/bcc/issues/1883. + data.ppid = task->real_parent->tgid; + + bpf_get_current_comm(&data.comm, sizeof(data.comm)); + data.type = EVENT_RET; + data.retval = PT_REGS_RC(ctx); + execve_events.perf_submit(ctx, &data, sizeof(data)); + + return 0; +}` diff --git a/lib/bpf/common.go b/lib/bpf/common.go new file mode 100644 index 00000000000..0c14a62a510 --- /dev/null +++ b/lib/bpf/common.go @@ -0,0 +1,160 @@ +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +// #cgo LDFLAGS: -ldl +// #include +// #include +import "C" + +import ( + "unsafe" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/events" + "github.com/gravitational/teleport/lib/utils" + + "github.com/gravitational/trace" + + "github.com/coreos/go-semver/semver" +) + +// BPF implements an interface to open and close a recording session. +type BPF interface { + // OpenSession will start monitoring all events within a session and + // emitting them to the Audit Log. + OpenSession(ctx *SessionContext) (uint64, error) + + // CloseSession will stop monitoring events for a particular session. + CloseSession(ctx *SessionContext) error + + // Close will stop any running BPF programs. + Close() error +} + +// SessionContext contains all the information needed to track and emit +// events for a particular session. Most of this information is already within +// srv.ServerContext, unfortunately due to circular imports with lib/srv and +// lib/bpf, part of that structure is reproduced in SessionContext. +type SessionContext struct { + // Namespace is the namespace within which this session occurs. + Namespace string + + // SessionID is the UUID of the given session. + SessionID string + + // ServerID is the UUID of the server this session is executing on. + ServerID string + + // Login is the Unix login for this session. + Login string + + // User is the Teleport user. + User string + + // PID is the process ID of Teleport when it re-executes itself. This is + // used by Teleport to find itself by cgroup. + PID int + + // AuditLog is used to store events for a particular sessionl + AuditLog events.IAuditLog + + // Events is the set of events (command, disk, or network) to record for + // this session. + Events map[string]bool +} + +// Config holds configuration for the BPF service. +type Config struct { + // Enabled is if this service will try and install BPF programs on this system. + Enabled bool + + // CommandBufferSize is the size of the perf buffer for command events. + CommandBufferSize int + + // DiskBufferSize is the size of the perf buffer for disk events. + DiskBufferSize int + + // NetworkBufferSize is the size of the perf buffer for network events. + NetworkBufferSize int + + // CgroupPath is where the cgroupv2 hierarchy is mounted. + CgroupPath string +} + +// CheckAndSetDefaults checks BPF configuration. +func (c *Config) CheckAndSetDefaults() error { + if c.CommandBufferSize == 0 { + c.CommandBufferSize = defaults.PerfBufferPageCount + } + if c.DiskBufferSize == 0 { + c.DiskBufferSize = defaults.OpenPerfBufferPageCount + } + if c.NetworkBufferSize == 0 { + c.NetworkBufferSize = defaults.PerfBufferPageCount + } + if c.CgroupPath == "" { + c.CgroupPath = defaults.CgroupPath + } + + return nil +} + +// NOP is used on either non-Linux systems or when BPF support is not enabled. +type NOP struct { +} + +// Close will close the NOP service. Note this function does nothing. +func (s *NOP) Close() error { + return nil +} + +// OpenSession will open a NOP session. Note this function does nothing. +func (s *NOP) OpenSession(ctx *SessionContext) (uint64, error) { + return 0, nil +} + +// OpenSession will open a NOP session. Note this function does nothing. +func (s *NOP) CloseSession(ctx *SessionContext) error { + return nil +} + +// IsHostCompatible checks that BPF programs can run on this host. +func IsHostCompatible() error { + // To find the cgroup ID of a program, bpf_get_current_cgroup_id is needed + // which was introduced in 4.18. + // https://github.com/torvalds/linux/commit/bf6fa2c893c5237b48569a13fa3c673041430b6c + minKernel := semver.New(teleport.EnhancedRecordingMinKernel) + version, err := utils.KernelVersion() + if err != nil { + return trace.Wrap(err) + } + if version.LessThan(*minKernel) { + return trace.BadParameter("incompatible kernel found, minimum supported kernel is %v", minKernel) + } + + // Check that libbcc is on the system. + libraryName := C.CString("libbcc.so.0") + defer C.free(unsafe.Pointer(libraryName)) + handle := C.dlopen(libraryName, C.RTLD_NOW) + if handle == nil { + return trace.BadParameter("libbcc.so not found") + } + + return nil +} diff --git a/lib/bpf/disk.go b/lib/bpf/disk.go new file mode 100644 index 00000000000..93a75758cfd --- /dev/null +++ b/lib/bpf/disk.go @@ -0,0 +1,207 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +import "C" + +import ( + "context" + + "github.com/gravitational/teleport" + + "github.com/gravitational/trace" + + "github.com/iovisor/gobpf/bcc" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + lostDiskEvents = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: teleport.MetricLostDiskEvents, + Help: "Number of lost disk events.", + }, + ) +) + +func init() { + prometheus.MustRegister(lostDiskEvents) +} + +// rawOpenEvent is sent by the eBPF program that Teleport pulls off the perf +// buffer. +type rawOpenEvent struct { + // CgroupID is the internal cgroupv2 ID of the event. + CgroupID uint64 + + // PID is the ID of the process. + PID uint64 + + // ReturnCode is the return code of open. + ReturnCode int32 + + // Command is name of the executable opening the file. + Command [commMax]byte + + // Path is the full path to the file being opened. + Path [pathMax]byte + + // Flags are the flags passed to open. + Flags int32 +} + +// open runs a BPF program (opensnoop) that hooks openat. +type open struct { + closeContext context.Context + + eventCh <-chan []byte + lostCh <-chan uint64 + + perfMaps []*bcc.PerfMap + module *bcc.Module +} + +// startOpen will compile, load, start, and pull events off the perf buffer +// for the BPF program. +func startOpen(closeContext context.Context, pageCount int) (*open, error) { + var err error + + e := &open{ + closeContext: closeContext, + } + + // Compile the BPF program. + e.module = bcc.NewModule(openSource, []string{}) + if e.module == nil { + return nil, trace.BadParameter("failed to load libbcc") + } + + // Hook open syscall. + err = attachProbe(e.module, "do_sys_open", "trace_entry") + if err != nil { + return nil, trace.Wrap(err) + } + err = attachRetProbe(e.module, "do_sys_open", "trace_return") + if err != nil { + return nil, trace.Wrap(err) + } + + // Open perf buffer and start processing open events. + e.eventCh, e.lostCh, err = openPerfBuffer(e.module, e.perfMaps, pageCount, "open_events") + if err != nil { + return nil, trace.Wrap(err) + } + + // Start a loop that will emit lost events to prometheus. + go e.lostLoop() + + return e, nil +} + +// close will stop reading events off the perf buffer and unload the BPF +// program. +func (e *open) close() { + for _, perfMap := range e.perfMaps { + perfMap.Stop() + } + e.module.Close() +} + +// lostLoop keeps emitting the number of lost events to prometheus. +func (e *open) lostLoop() { + for { + select { + case n := <-e.lostCh: + log.Debugf("Lost %v disk events.", n) + lostDiskEvents.Add(float64(n)) + case <-e.closeContext.Done(): + return + } + } +} + +// events contains raw events off the perf buffer. +func (e *open) events() <-chan []byte { + return e.eventCh +} + +const openSource string = ` +#include +#include +#include +#include +#include + +struct val_t { + u64 pid; + char comm[TASK_COMM_LEN]; + const char *fname; + int flags; +}; + +struct data_t { + u64 cgroup; + u64 pid; + int ret; + char comm[TASK_COMM_LEN]; + char fname[NAME_MAX]; + int flags; +}; + +BPF_HASH(infotmp, u64, struct val_t); +BPF_PERF_OUTPUT(open_events); + +int trace_entry(struct pt_regs *ctx, int dfd, const char __user *filename, int flags) +{ + struct val_t val = {}; + u64 id = bpf_get_current_pid_tgid(); + + if (bpf_get_current_comm(&val.comm, sizeof(val.comm)) == 0) { + val.pid = id >> 32; + val.fname = filename; + val.flags = flags; + infotmp.update(&id, &val); + } + + return 0; +}; + +int trace_return(struct pt_regs *ctx) +{ + u64 id = bpf_get_current_pid_tgid(); + struct val_t *valp; + struct data_t data = {}; + + valp = infotmp.lookup(&id); + if (valp == 0) { + // Missed entry. + return 0; + } + bpf_probe_read(&data.comm, sizeof(data.comm), valp->comm); + bpf_probe_read(&data.fname, sizeof(data.fname), (void *)valp->fname); + data.pid = valp->pid; + data.flags = valp->flags; + data.ret = PT_REGS_RC(ctx); + data.cgroup = bpf_get_current_cgroup_id(); + + open_events.perf_submit(ctx, &data, sizeof(data)); + infotmp.delete(&id); + + return 0; +}` diff --git a/lib/bpf/helper.go b/lib/bpf/helper.go new file mode 100644 index 00000000000..548cca15f6b --- /dev/null +++ b/lib/bpf/helper.go @@ -0,0 +1,102 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +import ( + "github.com/gravitational/teleport" + + "github.com/gravitational/trace" + + "github.com/iovisor/gobpf/bcc" + "github.com/sirupsen/logrus" +) + +var log = logrus.WithFields(logrus.Fields{ + trace.Component: teleport.ComponentBPF, +}) + +// attachProbe will attach a kprobe to the given function name. +func attachProbe(module *bcc.Module, eventName string, functionName string) error { + kprobe, err := module.LoadKprobe(functionName) + if err != nil { + return trace.Wrap(err) + } + + err = module.AttachKprobe(eventName, kprobe, -1) + if err != nil { + return trace.Wrap(err) + } + + return nil +} + +// attachRetProbe will attach a kretprobe to the given function name. +func attachRetProbe(module *bcc.Module, eventName string, functionName string) error { + kretprobe, err := module.LoadKprobe(functionName) + if err != nil { + return trace.Wrap(err) + } + + err = module.AttachKretprobe(eventName, kretprobe, -1) + if err != nil { + return trace.Wrap(err) + } + + return nil +} + +// openPerfBuffer will open a perf buffer for a particular module. +func openPerfBuffer(module *bcc.Module, perfMaps []*bcc.PerfMap, pageCount int, name string) (<-chan []byte, <-chan uint64, error) { + var err error + + eventCh := make(chan []byte, chanSize) + lostCh := make(chan uint64, chanSize) + + table := bcc.NewTable(module.TableId(name), module) + + perfMap, err := bcc.InitPerfMap(table, eventCh, lostCh, uint(pageCount)) + if err != nil { + return nil, nil, trace.Wrap(err) + } + perfMap.Start() + + perfMaps = append(perfMaps, perfMap) + + return eventCh, lostCh, nil +} + +const ( + // commMax is the maximum length of a command from linux/sched.h. + commMax = 16 + + // pathMax is the maximum length of a path from linux/limits.h. + pathMax = 255 + + // argvMax is the maximum length of the args vector. + argvMax = 128 + + // eventArg is an exec event that holds the arguments to a function. + eventArg = 0 + + // eventRet holds the return value and other data about about an event. + eventRet = 1 + + // chanSize is the size of the event and lost event channels. + chanSize = 1024 +) diff --git a/lib/bpf/network.go b/lib/bpf/network.go new file mode 100644 index 00000000000..7e8c1487b73 --- /dev/null +++ b/lib/bpf/network.go @@ -0,0 +1,312 @@ +// +build bpf,!386 + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bpf + +import "C" + +import ( + "context" + + "github.com/gravitational/teleport" + + "github.com/gravitational/trace" + + "github.com/iovisor/gobpf/bcc" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + lostNetworkEvents = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: teleport.MetricLostNetworkEvents, + Help: "Number of lost network events.", + }, + ) +) + +func init() { + prometheus.MustRegister(lostNetworkEvents) +} + +// rawConn4Event is sent by the eBPF program that Teleport pulls off the perf +// buffer. +type rawConn4Event struct { + // CgroupID is the internal cgroupv2 ID of the event. + CgroupID uint64 + + // Version is the version of TCP (4 or 6). + Version uint64 + + // PID is the process ID. + PID uint32 + + // SrcAddr is the source IP address. + SrcAddr uint32 + + // DstAddr is the destination IP address. + DstAddr uint32 + + // DstPort is the port the connection is being made to. + DstPort uint16 + + // Command is name of the executable making the connection. + Command [commMax]byte +} + +// rawConn6Event is sent by the eBPF program that Teleport pulls off the perf +// buffer. +type rawConn6Event struct { + // CgroupID is the internal cgroupv2 ID of the event. + CgroupID uint64 + + // Version is the version of TCP (4 or 6). + Version uint64 + + // PID is the process ID. + PID uint32 + + // SrcAddr is the source IP address. + SrcAddr [4]uint32 + + // DstAddr is the destination IP address. + DstAddr [4]uint32 + + // DstPort is the port the connection is being made to. + DstPort uint16 + + // Command is name of the executable making the connection. + Command [commMax]byte +} + +type conn struct { + closeContext context.Context + + // v{4,6}EventCh are the channels upon which the perf buffer places + // events. + v4EventCh <-chan []byte + v6EventCh <-chan []byte + + // v{4,6}LostCh are the channels upon which the perf buffer places lost + // event count. + v4LostCh <-chan uint64 + v6LostCh <-chan uint64 + + module *bcc.Module + perfMaps []*bcc.PerfMap +} + +func startConn(closeContext context.Context, pageCount int) (*conn, error) { + var err error + + e := &conn{ + closeContext: closeContext, + } + + e.module = bcc.NewModule(connSource, []string{}) + if e.module == nil { + return nil, trace.BadParameter("failed to load libbcc") + } + + // Hook IPv4 connection attempts. + err = attachProbe(e.module, "tcp_v4_connect", "trace_connect_entry") + if err != nil { + return nil, trace.Wrap(err) + } + err = attachRetProbe(e.module, "tcp_v4_connect", "trace_connect_v4_return") + if err != nil { + return nil, trace.Wrap(err) + } + + // Hook IPv6 connection attempts. + err = attachProbe(e.module, "tcp_v6_connect", "trace_connect_entry") + if err != nil { + return nil, trace.Wrap(err) + } + err = attachRetProbe(e.module, "tcp_v6_connect", "trace_connect_v6_return") + if err != nil { + return nil, trace.Wrap(err) + } + + // Open perf buffer and start processing IPv4 events. + e.v4EventCh, e.v4LostCh, err = openPerfBuffer(e.module, e.perfMaps, pageCount, "ipv4_events") + if err != nil { + return nil, trace.Wrap(err) + } + + // Open perf buffer and start processing IPv6 events. + e.v6EventCh, e.v6LostCh, err = openPerfBuffer(e.module, e.perfMaps, pageCount, "ipv6_events") + if err != nil { + return nil, trace.Wrap(err) + } + + // Start a loop that will emit lost events to prometheus. + go e.lostLoop() + + return e, nil +} + +// close will stop reading events off the perf buffer and unload the BPF +// program. +func (e *conn) close() { + for _, perfMap := range e.perfMaps { + perfMap.Stop() + } + e.module.Close() +} + +// lostLoop keeps emitting the number of lost events to prometheus. +func (e *conn) lostLoop() { + for { + select { + case n := <-e.v4LostCh: + log.Debugf("Lost %v IPv4 events.", n) + lostNetworkEvents.Add(float64(n)) + case n := <-e.v6LostCh: + log.Debugf("Lost %v IPv6 events.", n) + lostNetworkEvents.Add(float64(n)) + case <-e.closeContext.Done(): + return + } + } +} + +// v4Events contains raw events off the perf buffer. +func (e *conn) v4Events() <-chan []byte { + return e.v4EventCh +} + +// v6Events contains raw events off the perf buffer. +func (e *conn) v6Events() <-chan []byte { + return e.v6EventCh +} + +const connSource string = ` +#include +#include +#include + +BPF_HASH(currsock, u32, struct sock *); + +// separate data structs for ipv4 and ipv6 +struct ipv4_data_t { + u64 cgroup; + u64 ip; + u32 pid; + u32 saddr; + u32 daddr; + u16 dport; + char task[TASK_COMM_LEN]; +}; +BPF_PERF_OUTPUT(ipv4_events); + +struct ipv6_data_t { + u64 cgroup; + u64 ip; + u32 pid; + u32 saddr[4]; + u32 daddr[4]; + u16 dport; + char task[TASK_COMM_LEN]; +}; +BPF_PERF_OUTPUT(ipv6_events); + +int trace_connect_entry(struct pt_regs *ctx, struct sock *sk) +{ + u32 pid = bpf_get_current_pid_tgid(); + + // Stash the sock ptr for lookup on return. + currsock.update(&pid, &sk); + + return 0; +}; + +static int trace_connect_return(struct pt_regs *ctx, short ipver) +{ + int ret = PT_REGS_RC(ctx); + u32 pid = bpf_get_current_pid_tgid(); + + struct sock **skpp; + skpp = currsock.lookup(&pid); + if (skpp == 0) { + return 0; // missed entry + } + + if (ret != 0) { + // failed to send SYNC packet, may not have populated + // socket __sk_common.{skc_rcv_saddr, ...} + currsock.delete(&pid); + return 0; + } + + // pull in details + struct sock *skp = *skpp; + u16 dport = skp->__sk_common.skc_dport; + + if (ipver == 4) { + struct ipv4_data_t data4 = {.pid = pid, .ip = ipver}; + data4.saddr = skp->__sk_common.skc_rcv_saddr; + data4.daddr = skp->__sk_common.skc_daddr; + data4.dport = ntohs(dport); + data4.cgroup = bpf_get_current_cgroup_id(); + bpf_get_current_comm(&data4.task, sizeof(data4.task)); + ipv4_events.perf_submit(ctx, &data4, sizeof(data4)); + + } else /* 6 */ { + struct ipv6_data_t data6 = {.pid = pid, .ip = ipver}; + + // Source. + bpf_probe_read(&data6.saddr[0], sizeof(data6.saddr[0]), + &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[0]); + bpf_probe_read(&data6.saddr[1], sizeof(data6.saddr[1]), + &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[1]); + bpf_probe_read(&data6.saddr[2], sizeof(data6.saddr[2]), + &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[2]); + bpf_probe_read(&data6.saddr[3], sizeof(data6.saddr[3]), + &skp->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32[3]); + + // Destination. + bpf_probe_read(&data6.daddr[0], sizeof(data6.daddr[0]), + &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[0]); + bpf_probe_read(&data6.daddr[1], sizeof(data6.daddr[1]), + &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[1]); + bpf_probe_read(&data6.daddr[2], sizeof(data6.daddr[2]), + &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[2]); + bpf_probe_read(&data6.daddr[3], sizeof(data6.daddr[3]), + &skp->__sk_common.skc_v6_daddr.in6_u.u6_addr32[3]); + + data6.dport = ntohs(dport); + data6.cgroup = bpf_get_current_cgroup_id(); + bpf_get_current_comm(&data6.task, sizeof(data6.task)); + ipv6_events.perf_submit(ctx, &data6, sizeof(data6)); + } + + currsock.delete(&pid); + + return 0; +} + +int trace_connect_v4_return(struct pt_regs *ctx) +{ + return trace_connect_return(ctx, 4); +} + +int trace_connect_v6_return(struct pt_regs *ctx) +{ + return trace_connect_return(ctx, 6); +}` diff --git a/lib/cgroup/cgroup.c b/lib/cgroup/cgroup.c new file mode 100644 index 00000000000..787ae2c7784 --- /dev/null +++ b/lib/cgroup/cgroup.c @@ -0,0 +1,55 @@ +// +build linux + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +// cgid_file_handle comes from bpftrace, see: +// https://github.com/iovisor/bpftrace/blob/master/src/resolve_cgroupid.cpp +struct cgid_file_handle { + unsigned int handle_bytes; + int handle_type; + uint64_t cgid; +}; + +// cgroup_id returns the ID of the given cgroup at path. +uint64_t cgroup_id(char *path) +{ + int ret; + int mount_id; + struct cgid_file_handle *handle; + + handle = malloc(sizeof(struct cgid_file_handle)); + if (handle == NULL) { + return 0; + } + handle->handle_bytes = sizeof(uint64_t); + + ret = name_to_handle_at(AT_FDCWD, path, (struct file_handle *)handle, &mount_id, 0); + if (ret != 0) { + return 0; + } + + free(handle); + + return handle->cgid; +} diff --git a/lib/cgroup/cgroup.go b/lib/cgroup/cgroup.go new file mode 100644 index 00000000000..1aaf3443038 --- /dev/null +++ b/lib/cgroup/cgroup.go @@ -0,0 +1,362 @@ +// +build linux + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cgroup + +// #include +// #include +// extern uint64_t cgroup_id(char *path); +import "C" + +import ( + "bufio" + "io/ioutil" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "unsafe" + + "golang.org/x/sys/unix" + + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/defaults" + + "github.com/gravitational/trace" + + "github.com/pborman/uuid" + "github.com/sirupsen/logrus" +) + +var log = logrus.WithFields(logrus.Fields{ + trace.Component: teleport.ComponentCgroup, +}) + +// Config holds configuration for the cgroup service. +type Config struct { + // MountPath is where the cgroupv2 hierarchy is mounted. + MountPath string +} + +// CheckAndSetDefaults checks BPF configuration. +func (c *Config) CheckAndSetDefaults() error { + if c.MountPath == "" { + c.MountPath = defaults.CgroupPath + } + return nil +} + +// Service manages cgroup orchestration. +type Service struct { + *Config + + // teleportRoot is the root cgroup that holds all Teleport sessions. Used + // to remove all cgroups upon shutdown. + teleportRoot string +} + +// New creates a new cgroup service. +func New(config *Config) (*Service, error) { + err := config.CheckAndSetDefaults() + if err != nil { + return nil, trace.Wrap(err) + } + + s := &Service{ + Config: config, + teleportRoot: path.Join(config.MountPath, teleportRoot, uuid.New()), + } + + // Mount the cgroup2 filesystem. + err = s.mount() + if err != nil { + return nil, trace.Wrap(err) + } + + log.Debugf("Teleport session hierarchy mounted at: %v.", s.teleportRoot) + + return s, nil +} + +// Close will unmount the cgroup filesystem. +func (s *Service) Close() error { + err := s.cleanupHierarchy() + if err != nil { + return trace.Wrap(err) + } + + err = s.unmount() + if err != nil { + return trace.Wrap(err) + } + + log.Debugf("Cleaned up and unmounted Teleport session hierarchy at: %v.", s.teleportRoot) + return nil +} + +// Create will create a cgroup for a given session. +func (s *Service) Create(sessionID string) error { + err := os.Mkdir(path.Join(s.teleportRoot, sessionID), fileMode) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// Remove will remove the cgroup for a session. An existing processes will be +// moved to the root controller. +func (s *Service) Remove(sessionID string) error { + // Read in all PIDs for the cgroup. + pids, err := readPids(path.Join(s.teleportRoot, sessionID, cgroupProcs)) + if err != nil { + return trace.Wrap(err) + } + + // Move all PIDs to the root controller. This has to be done before a cgroup + // can be removed. + err = writePids(path.Join(s.MountPath, cgroupProcs), pids) + if err != nil { + return trace.Wrap(err) + } + + // The rmdir syscall is used to remove a cgroup. + err = unix.Rmdir(path.Join(s.teleportRoot, sessionID)) + if err != nil { + return trace.Wrap(err) + } + + log.Debugf("Removed cgroup for session: %v.", sessionID) + + return nil +} + +// Place place a process in the cgroup for that session. +func (s *Service) Place(sessionID string, pid int) error { + // Open cgroup.procs file for the cgroup. + filepath := path.Join(s.teleportRoot, sessionID, cgroupProcs) + f, err := os.OpenFile(filepath, os.O_APPEND|os.O_WRONLY, fileMode) + if err != nil { + return trace.Wrap(err) + } + defer f.Close() + + // Write PID and place process in cgroup. + _, err = f.WriteString(strconv.Itoa(pid)) + if err != nil { + return trace.Wrap(err) + } + + return nil +} + +// readPids returns a slice of PIDs from a file. Used to get list of all PIDs +// within a cgroup. +func readPids(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, trace.Wrap(err) + } + defer f.Close() + + var pids []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + pids = append(pids, scanner.Text()) + } + if scanner.Err() != nil { + return nil, trace.Wrap(err) + } + + return pids, nil +} + +// writePids writes a slice of PIDS to a given file. Used to add processes to +// a cgroup. +func writePids(path string, pids []string) error { + f, err := os.OpenFile(path, os.O_WRONLY, fileMode) + if err != nil { + return trace.Wrap(err) + } + defer f.Close() + + for _, pid := range pids { + _, err := f.WriteString(pid + "\n") + if err != nil { + return trace.Wrap(err) + } + } + + return nil +} + +// cleanupHierarchy removes any cgroups for any exisiting sessions. +func (s *Service) cleanupHierarchy() error { + var sessions []string + + // Recursively look within the Teleport hierarchy for cgroups for session. + err := filepath.Walk(path.Join(s.teleportRoot), func(path string, info os.FileInfo, err error) error { + // Only pick up cgroup.procs files. + if !pattern.MatchString(path) { + return nil + } + + // Extract the session ID. Skip over cgroup.procs files not for sessions. + parts := strings.Split(path, string(filepath.Separator)) + if len(parts) != 5 { + return nil + } + sessionID := uuid.Parse(parts[3]) + if sessionID == nil { + return nil + } + + // Append to the list of sessions within the cgroup hierarchy. + sessions = append(sessions, sessionID.String()) + + return nil + }) + if err != nil { + return trace.Wrap(err) + } + + // Remove all sessions that were found. + for _, sessionID := range sessions { + err := s.Remove(sessionID) + if err != nil { + return trace.Wrap(err) + } + } + + return nil +} + +// mount mounts the cgroup2 filesystem. +func (s *Service) mount() error { + // Make sure path to cgroup2 mount point exists. + err := os.MkdirAll(s.MountPath, fileMode) + if err != nil { + return trace.Wrap(err) + } + + // Check if the Teleport root cgroup exists, if it does the cgroup filesystem + // is already mounted, return right away. + files, err := ioutil.ReadDir(s.MountPath) + if err == nil && len(files) > 0 { + // Create cgroup that will hold Teleport sessions. + err = os.MkdirAll(s.teleportRoot, fileMode) + if err != nil { + return trace.Wrap(err) + } + return nil + } + + // Mount the cgroup2 filesystem. Even if the cgroup filesystem is already + // mounted, it is safe to re-mount it at another location, both will have + // the exact same view of the hierarchy. From "man cgroups": + // + // It is not possible to mount the same controller against multiple + // cgroup hierarchies. For example, it is not possible to mount both + // the cpu and cpuacct controllers against one hierarchy, and to mount + // the cpu controller alone against another hierarchy. It is possible + // to create multiple mount points with exactly the same set of + // comounted controllers. However, in this case all that results is + // multiple mount points providing a view of the same hierarchy. + // + // The exact args to the mount syscall come strace of mount(8). From the + // docs: https://www.kernel.org/doc/Documentation/cgroup-v2.txt: + // + // Unlike v1, cgroup v2 has only single hierarchy. The cgroup v2 + // hierarchy can be mounted with the following mount command: + // + // # mount -t cgroup2 none $MOUNT_POINT + // + // The output of the strace looks like the following: + // + // mount("none", "/cgroup3", "cgroup2", MS_MGC_VAL, NULL) = 0 + // + // Where MS_MGC_VAL can be dropped. From mount(2) because we only support + // kernels 4.18 and above for this feature. + // + // The mountflags argument may have the magic number 0xC0ED (MS_MGC_VAL) + // in the top 16 bits. (All of the other flags discussed in DESCRIPTION + // occupy the low order 16 bits of mountflags.) Specifying MS_MGC_VAL + // was required in kernel versions prior to 2.4, but since Linux 2.4 is + // no longer required and is ignored if specified. + err = unix.Mount("none", s.MountPath, "cgroup2", 0, "") + if err != nil { + return trace.Wrap(err) + } + log.Debugf("Mounted cgroup filesystem to %v.", s.MountPath) + + // Create cgroup that will hold Teleport sessions. + err = os.MkdirAll(s.teleportRoot, fileMode) + if err != nil { + return trace.Wrap(err) + } + + return nil +} + +// unmount will unmount the cgroupv2 filesystem. +func (s *Service) unmount() error { + // The exact args to the umount syscall come from a strace of umount(8): + // + // umount2("/cgroup2", 0) = 0 + err := unix.Unmount(s.MountPath, 0) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// ID returns the cgroup ID for the given session. +func (s *Service) ID(sessionID string) (uint64, error) { + path := path.Join(s.teleportRoot, sessionID) + + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + + // Returns the cgroup ID of a given path. + cgid := C.cgroup_id(cpath) + if cgid == 0 { + return 0, trace.BadParameter("cgroup resolution failed") + } + + return uint64(cgid), nil +} + +var ( + // pattern matches cgroup process files. + pattern = regexp.MustCompile(`cgroup\.procs$`) +) + +const ( + // fileMode is the mode files and directories are created in within the + // cgroup filesystem. + fileMode = 0555 + + // teleportRoot is the prefix of the root cgroup that holds all other + // Teleport cgroups. + teleportRoot = "teleport" + + // cgroupProcs is the name of the file that contains all processes within + // a cgroup. + cgroupProcs = "cgroup.procs" +) diff --git a/lib/cgroup/cgroup_test.go b/lib/cgroup/cgroup_test.go new file mode 100644 index 00000000000..614f179b6ab --- /dev/null +++ b/lib/cgroup/cgroup_test.go @@ -0,0 +1,146 @@ +// +build linux + +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cgroup + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/gravitational/teleport/lib/utils" + + "github.com/pborman/uuid" + "gopkg.in/check.v1" +) + +type Suite struct{} + +var _ = fmt.Printf +var _ = check.Suite(&Suite{}) + +func TestControlGroups(t *testing.T) { check.TestingT(t) } + +func (s *Suite) SetUpSuite(c *check.C) { + utils.InitLoggerForTests() +} +func (s *Suite) TearDownSuite(c *check.C) {} +func (s *Suite) SetUpTest(c *check.C) {} +func (s *Suite) TearDownTest(c *check.C) {} + +// TestCreate tests creating and removing cgroups as well as shutting down +// the service and unmounting the cgroup hierarchy. +func (s *Suite) TestCreate(c *check.C) { + // This test must be run as root. Only root can create cgroups. + if !isRoot() { + c.Skip("Tests for package cgroup can only be run as root.") + } + + // Create temporary directory where cgroup2 hierarchy will be mounted. + dir, err := ioutil.TempDir("", "cgroup-test") + c.Assert(err, check.IsNil) + defer os.RemoveAll(dir) + + // Start cgroup service. + service, err := New(&Config{ + MountPath: dir, + }) + c.Assert(err, check.IsNil) + + // Create fake session ID and cgroup. + sessionID := uuid.New() + err = service.Create(sessionID) + c.Assert(err, check.IsNil) + + // Make sure that it exists. + cgroupPath := path.Join(service.teleportRoot, sessionID) + _, err = os.Stat(cgroupPath) + if os.IsNotExist(err) { + c.Fatalf("Could not find cgroup file %v: %v.", cgroupPath, err) + } + + // Remove cgroup. + err = service.Remove(sessionID) + c.Assert(err, check.IsNil) + + // Make sure cgroup is gone. + _, err = os.Stat(cgroupPath) + if !os.IsNotExist(err) { + c.Fatalf("Failed to remove cgroup at %v: %v.", cgroupPath, err) + } + + // Close the cgroup service, this should unmound the cgroup filesystem. + err = service.Close() + c.Assert(err, check.IsNil) + + // Make sure the cgroup filesystem has been unmounted. + _, err = os.Stat(cgroupPath) + if !os.IsNotExist(err) { + c.Fatalf("Failed to unmound cgroup filesystem from %v: %v.", dir, err) + } +} + +// TestCleanup tests the ability for Teleport to remove and cleanup all +// cgroups which is performed upon startup. +func (s *Suite) TestCleanup(c *check.C) { + // This test must be run as root. Only root can create cgroups. + if !isRoot() { + c.Skip("Tests for package cgroup can only be run as root.") + } + + // Create temporary directory where cgroup2 hierarchy will be mounted. + dir, err := ioutil.TempDir("", "cgroup-test") + c.Assert(err, check.IsNil) + if err != nil { + } + defer os.RemoveAll(dir) + + // Start cgroup service. + service, err := New(&Config{ + MountPath: dir, + }) + defer service.Close() + c.Assert(err, check.IsNil) + + // Create fake session ID and cgroup. + sessionID := uuid.New() + err = service.Create(sessionID) + c.Assert(err, check.IsNil) + + // Cleanup hierarchy to remove all cgroups. + err = service.cleanupHierarchy() + c.Assert(err, check.IsNil) + + // Make sure the cgroup no longer exists. + cgroupPath := path.Join(service.teleportRoot, sessionID) + _, err = os.Stat(cgroupPath) + if os.IsNotExist(err) { + c.Fatalf("Could not find cgroup file %v: %v.", cgroupPath, err) + } +} + +// isRoot returns a boolean if the test is being run as root or not. Tests +// for this package must be run as root. +func isRoot() bool { + if os.Geteuid() != 0 { + return false + } + return true +} diff --git a/lib/config/configuration.go b/lib/config/configuration.go index 7dbdc166262..2d8b1c93e9e 100644 --- a/lib/config/configuration.go +++ b/lib/config/configuration.go @@ -632,6 +632,9 @@ func applySSHConfig(fc *FileConfig, cfg *service.Config) error { } cfg.SSH.PublicAddrs = addrs } + if fc.SSH.BPF != nil { + cfg.SSH.BPF = fc.SSH.BPF.Parse() + } return nil } diff --git a/lib/config/fileconf.go b/lib/config/fileconf.go index e812feefb4f..edd19d00f9c 100644 --- a/lib/config/fileconf.go +++ b/lib/config/fileconf.go @@ -30,6 +30,7 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/backend" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/pam" "github.com/gravitational/teleport/lib/service" @@ -150,6 +151,11 @@ var ( "keep_alive_interval": false, "keep_alive_count_max": false, "local_auth": false, + "enhanced_recording": false, + "command_buffer_size": false, + "disk_buffer_size": false, + "network_buffer_size": false, + "cgroup_path": false, } ) @@ -679,6 +685,9 @@ type SSH struct { PAM *PAM `yaml:"pam,omitempty"` // PublicAddr sets SSH host principals for SSH service PublicAddr utils.Strings `yaml:"public_addr,omitempty"` + + // BPF is used to configure BPF-based auditing for this node. + BPF *BPF `yaml:"enhanced_recording,omitempty"` } // CommandLabel is `command` section of `ssh_service` in the config file @@ -710,6 +719,36 @@ func (p *PAM) Parse() *pam.Config { } } +// BPF is configuration for BPF-based auditing. +type BPF struct { + // Enabled enables or disables enhanced session recording for this node. + Enabled string `yaml:"enabled"` + + // CommandBufferSize is the size of the perf buffer for command events. + CommandBufferSize int `yaml:"command_buffer_size"` + + // DiskBufferSize is the size of the perf buffer for disk events. + DiskBufferSize int `yaml:"disk_buffer_size"` + + // NetworkBufferSize is the size of the perf buffer for network events. + NetworkBufferSize int `yaml:"network_buffer_size"` + + // CgroupPath controls where cgroupv2 hierarchy is mounted. + CgroupPath string `yaml:"cgroup_path"` +} + +// Parse will parse the enhanced session recording configuration. +func (b *BPF) Parse() *bpf.Config { + enabled, _ := utils.ParseBool(b.Enabled) + return &bpf.Config{ + Enabled: enabled, + CommandBufferSize: b.CommandBufferSize, + DiskBufferSize: b.DiskBufferSize, + NetworkBufferSize: b.NetworkBufferSize, + CgroupPath: b.CgroupPath, + } +} + // Proxy is a `proxy_service` section of the config file: type Proxy struct { // Service is a generic service configuration section diff --git a/lib/defaults/defaults.go b/lib/defaults/defaults.go index 6d87ab129a0..a23ff1661a4 100644 --- a/lib/defaults/defaults.go +++ b/lib/defaults/defaults.go @@ -23,6 +23,7 @@ import ( "fmt" "time" + "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/limiter" "github.com/gravitational/teleport/lib/utils" ) @@ -91,6 +92,9 @@ const ( // HTTPMaxIdleConnsPerHost is the max idle connections per-host. HTTPMaxIdleConnsPerHost = 1000 + // HTTPMaxConnsPerHost is the maximum number of connections per-host. + HTTPMaxConnsPerHost = 250 + // HTTPIdleTimeout is a default timeout for idle HTTP connections HTTPIdleTimeout = 30 * time.Second @@ -386,6 +390,32 @@ const ( RoleAuthService = "auth" ) +const ( + // PerfBufferPageCount is the size of the perf ring buffer in number of pages. + // Must be power of 2. + PerfBufferPageCount = 8 + + // OpenPerfBufferPageCount is the page count for the perf buffer. Open + // events generate many events so this buffer needs to be extra large. + // Must be power of 2. + OpenPerfBufferPageCount = 128 + + // CgroupPath is where the cgroupv2 hierarchy will be mounted. + CgroupPath = "/cgroup2" + + // ArgsCacheSize is the number of args events to store before dropping args + // events. + ArgsCacheSize = 1024 +) + +// EnhancedEvents returns the default list of enhanced events. +func EnhancedEvents() []string { + return []string{ + teleport.EnhancedRecordingCommand, + teleport.EnhancedRecordingNetwork, + } +} + var ( // ConfigFilePath is default path to teleport config file ConfigFilePath = "/etc/teleport.yaml" diff --git a/lib/events/api.go b/lib/events/api.go index a0eda28d820..42fb898ca85 100644 --- a/lib/events/api.go +++ b/lib/events/api.go @@ -76,14 +76,37 @@ const ( // SessionEndEvent indicates that a session has ended SessionEndEvent = "session.end" + // SessionUploadEvent indicates that session has been uploaded to the external storage SessionUploadEvent = "session.upload" + // URL is used for a session upload URL URL = "url" - SessionEventID = "sid" + // SessionEventID is a unique UUID of the session. + SessionEventID = "sid" + + // SessionServerID is the UUID of the server the session occurred on. SessionServerID = "server_id" + // SessionServerHostname is the hostname of the server the session occurred on. + SessionServerHostname = "server_hostname" + + // SessionEnhancedRecording is used to indicate if the recording was an + // enhanced recording or not. + SessionEnhancedRecording = "enhanced_recording" + + // SessionInteractive is used to indicate if the session was interactive + // (has PTY attached) or not (exec session). + SessionInteractive = "interactive" + + // SessionParticipants is a list of participants in the session. + SessionParticipants = "participants" + + // SessionServerLabels are the labels (static and dynamic) of the server the + // session occurred on. + SessionServerLabels = "server_labels" + // SessionByteOffset is the number of bytes written to session stream since // the beginning SessionByteOffset = "offset" @@ -195,6 +218,53 @@ const ( // to indicate one of the last session events, used to report // data transfer SessionDataIndex = math.MaxInt32 - 1 + + // SessionCommandEvent is emitted when an executable is run within a session. + SessionCommandEvent = "session.command" + + // SessionDiskEvent is emitted when a file is opened within an session. + SessionDiskEvent = "session.disk" + + // SessionNetworkEvent is emitted when a network connection is initated with a + // session. + SessionNetworkEvent = "session.network" + + // PID is the ID of the process. + PID = "pid" + + // PPID is the PID of the parent process. + PPID = "ppid" + + // CgroupID is the internal cgroupv2 ID of the event. + CgroupID = "cgroup_id" + + // Program is name of the executable. + Program = "program" + + // Path is the full path to the executable. + Path = "path" + + // Argv is the list of arguments to the program. Note, the first element does + // not contain the name of the process. + Argv = "argv" + + // ReturnCode is the return code of execve. + ReturnCode = "return_code" + + // Flags are the flags passed to open. + Flags = "flags" + + // SrcAddr is the source IP address of the connection. + SrcAddr = "src_addr" + + // DstAddr is the destination IP address of the connection. + DstAddr = "dst_addr" + + // DstPort is the destination port of the connection. + DstPort = "dst_port" + + // TCPVersion is the version of TCP (4 or 6). + TCPVersion = "version" ) const ( diff --git a/lib/events/auditlog.go b/lib/events/auditlog.go index 1953a8e68c4..635b1be27c6 100644 --- a/lib/events/auditlog.go +++ b/lib/events/auditlog.go @@ -419,12 +419,13 @@ func (l *AuditLog) getAuthServers() ([]string, error) { } type sessionIndex struct { - dataDir string - namespace string - sid session.ID - events []indexEntry - chunks []indexEntry - indexFiles []string + dataDir string + namespace string + sid session.ID + events []indexEntry + enhancedEvents map[string][]indexEntry + chunks []indexEntry + indexFiles []string } func (idx *sessionIndex) fileNames() []string { @@ -439,6 +440,13 @@ func (idx *sessionIndex) fileNames() []string { files = append(files, idx.chunksFileName(i)) } + // Enhanced events. + for k, v := range idx.enhancedEvents { + for i := range v { + files = append(files, idx.enhancedFileName(i, k)) + } + } + return files } @@ -449,6 +457,18 @@ func (idx *sessionIndex) sort() { sort.Slice(idx.chunks, func(i, j int) bool { return idx.chunks[i].Offset < idx.chunks[j].Offset }) + + // Enhanced events. + for k, _ := range idx.enhancedEvents { + sort.Slice(idx.enhancedEvents[k], func(i, j int) bool { + return idx.enhancedEvents[k][i].Index < idx.enhancedEvents[k][j].Index + }) + } +} + +func (idx *sessionIndex) enhancedFileName(index int, eventType string) string { + entry := idx.enhancedEvents[eventType][index] + return filepath.Join(idx.dataDir, entry.authServer, SessionLogsDir, idx.namespace, entry.FileName) } func (idx *sessionIndex) eventsFileName(index int) string { @@ -506,6 +526,11 @@ func readSessionIndex(dataDir string, authServers []string, namespace string, si sid: sid, dataDir: dataDir, namespace: namespace, + enhancedEvents: map[string][]indexEntry{ + SessionCommandEvent: []indexEntry{}, + SessionDiskEvent: []indexEntry{}, + SessionNetworkEvent: []indexEntry{}, + }, } for _, authServer := range authServers { indexFileName := filepath.Join(dataDir, authServer, SessionLogsDir, namespace, fmt.Sprintf("%v.index", sid)) @@ -518,12 +543,25 @@ func readSessionIndex(dataDir string, authServers []string, namespace string, si continue } index.indexFiles = append(index.indexFiles, indexFileName) - events, chunks, err := readIndexEntries(indexFile, authServer) + + entries, err := readIndexEntries(indexFile, authServer) if err != nil { return nil, trace.Wrap(err) } - index.events = append(index.events, events...) - index.chunks = append(index.chunks, chunks...) + for _, entry := range entries { + switch entry.Type { + case fileTypeEvents: + index.events = append(index.events, entry) + case fileTypeChunks: + index.chunks = append(index.chunks, entry) + // Enhanced events. + case SessionCommandEvent, SessionDiskEvent, SessionNetworkEvent: + index.enhancedEvents[entry.Type] = append(index.enhancedEvents[entry.Type], entry) + default: + return nil, trace.BadParameter("found unknown event type: %q", entry.Type) + } + } + err = indexFile.Close() if err != nil { return nil, trace.Wrap(err) @@ -533,24 +571,20 @@ func readSessionIndex(dataDir string, authServers []string, namespace string, si return &index, nil } -func readIndexEntries(file *os.File, authServer string) (events []indexEntry, chunks []indexEntry, err error) { +func readIndexEntries(file *os.File, authServer string) ([]indexEntry, error) { + var entries []indexEntry + scanner := bufio.NewScanner(file) for lineNo := 0; scanner.Scan(); lineNo++ { var entry indexEntry if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil { - return nil, nil, trace.Wrap(err) + return nil, trace.Wrap(err) } entry.authServer = authServer - switch entry.Type { - case fileTypeEvents: - events = append(events, entry) - case fileTypeChunks: - chunks = append(chunks, entry) - default: - return nil, nil, trace.BadParameter("unsupported type: %q", entry.Type) - } + entries = append(entries, entry) } - return + + return entries, nil } // createOrGetDownload creates a new download sync entry for a given session, diff --git a/lib/events/codes.go b/lib/events/codes.go index 4229e29ddad..c2969548772 100644 --- a/lib/events/codes.go +++ b/lib/events/codes.go @@ -159,6 +159,23 @@ var ( Name: AccessRequestUpdateEvent, Code: AccessRequestUpdateCode, } + // SessionCommand is emitted upon execution of a command when using enhanced + // session recording. + SessionCommand = Event{ + Name: SessionCommandEvent, + Code: SessionCommandCode, + } + // SessionDisk is emitted upon open of a file when using enhanced session recording. + SessionDisk = Event{ + Name: SessionDiskEvent, + Code: SessionDiskCode, + } + // SessionNetwork is emitted when a network requests is is issued when + // using enhanced session recording. + SessionNetwork = Event{ + Name: SessionNetworkEvent, + Code: SessionNetworkCode, + } ) var ( @@ -212,6 +229,12 @@ var ( ClientDisconnectCode = "T3006I" // AuthAttemptFailureCode is the auth attempt failure event code. AuthAttemptFailureCode = "T3007W" + // SessionCommandCode is a session command code. + SessionCommandCode = "T4000I" + // SessionDiskCode is a session disk code. + SessionDiskCode = "T4001I" + // SessionNetworkCode is a session network code. + SessionNetworkCode = "T4002I" // AccessRequestCreateCode is the the access request creation code. AccessRequestCreateCode = "T5000I" // AccessRequestUpdateCode is the access request state update code. diff --git a/lib/events/forward.go b/lib/events/forward.go index 4a1f3f08a77..85cece26974 100644 --- a/lib/events/forward.go +++ b/lib/events/forward.go @@ -86,6 +86,11 @@ func NewForwarder(cfg ForwarderConfig) (*Forwarder, error) { return &Forwarder{ ForwarderConfig: cfg, sessionLogger: diskLogger, + enhancedIndexes: map[string]int64{ + SessionCommandEvent: 0, + SessionDiskEvent: 0, + SessionNetworkEvent: 0, + }, }, nil } @@ -93,9 +98,10 @@ func NewForwarder(cfg ForwarderConfig) (*Forwarder, error) { // to the auth server, and writes the session playback to disk type Forwarder struct { ForwarderConfig - sessionLogger *DiskSessionLogger - lastChunk *SessionChunk - eventIndex int64 + sessionLogger *DiskSessionLogger + lastChunk *SessionChunk + eventIndex int64 + enhancedIndexes map[string]int64 sync.Mutex isClosed bool } @@ -150,6 +156,7 @@ func (l *Forwarder) PostSessionSlice(slice SessionSlice) error { if err != nil { return trace.Wrap(err) } + // no chunks to post (all chunks are print events) if len(chunksWithoutPrintEvents) == 0 { return nil @@ -168,17 +175,24 @@ func (l *Forwarder) setupSlice(slice *SessionSlice) ([]*SessionChunk, error) { return nil, trace.BadParameter("write on closed forwarder") } - // setup chunk indexes + // Setup chunk indexes. var chunks []*SessionChunk for _, chunk := range slice.Chunks { - chunk.EventIndex = l.eventIndex - l.eventIndex += 1 + switch chunk.EventType { case "": return nil, trace.BadParameter("missing event type") + case SessionCommandEvent, SessionDiskEvent, SessionNetworkEvent: + chunk.EventIndex = l.enhancedIndexes[chunk.EventType] + l.enhancedIndexes[chunk.EventType] += 1 + + chunks = append(chunks, chunk) case SessionPrintEvent: - // filter out chunks with session print events, - // as this logger forwards only audit events to the auth server + chunk.EventIndex = l.eventIndex + l.eventIndex += 1 + + // Filter out chunks with session print events, as this logger forwards + // only audit events to the auth server. if l.lastChunk != nil { chunk.Offset = l.lastChunk.Offset + int64(len(l.lastChunk.Data)) chunk.Delay = diff(time.Unix(0, l.lastChunk.Time), time.Unix(0, chunk.Time)) + l.lastChunk.Delay @@ -186,6 +200,9 @@ func (l *Forwarder) setupSlice(slice *SessionSlice) ([]*SessionChunk, error) { } l.lastChunk = chunk default: + chunk.EventIndex = l.eventIndex + l.eventIndex += 1 + chunks = append(chunks, chunk) } } diff --git a/lib/events/sessionlog.go b/lib/events/sessionlog.go index 5ed8790f5ef..1f8a5d54151 100644 --- a/lib/events/sessionlog.go +++ b/lib/events/sessionlog.go @@ -30,6 +30,7 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" @@ -102,6 +103,17 @@ func NewDiskSessionLogger(cfg DiskSessionLoggerConfig) (*DiskSessionLogger, erro indexFile: indexFile, lastEventIndex: -1, lastChunkIndex: -1, + + enhancedIndexes: map[string]int64{ + SessionCommandEvent: -1, + SessionDiskEvent: -1, + SessionNetworkEvent: -1, + }, + enhancedFiles: map[string]*gzipWriter{ + SessionCommandEvent: nil, + SessionDiskEvent: nil, + SessionNetworkEvent: nil, + }, } return sessionLogger, nil } @@ -125,6 +137,9 @@ type DiskSessionLogger struct { lastEventIndex int64 lastChunkIndex int64 + + enhancedIndexes map[string]int64 + enhancedFiles map[string]*gzipWriter } // LogEvent logs an event associated with this session @@ -170,15 +185,25 @@ func (sl *DiskSessionLogger) Finalize() error { // flush is used to flush gzip frames to file, otherwise // some attempts to read the file could fail func (sl *DiskSessionLogger) flush() error { - var err, err2 error + var err error + var errs []error if sl.RecordSessions && sl.chunksFile != nil { err = sl.chunksFile.Flush() + errs = append(errs, err) } if sl.eventsFile != nil { - err2 = sl.eventsFile.Flush() + err = sl.eventsFile.Flush() + errs = append(errs, err) } - return trace.NewAggregate(err, err2) + for _, eventsFile := range sl.enhancedFiles { + if eventsFile != nil { + err = eventsFile.Flush() + errs = append(errs, err) + } + } + + return trace.NewAggregate(errs...) } func (sl *DiskSessionLogger) finalize() error { @@ -201,6 +226,15 @@ func (sl *DiskSessionLogger) finalize() error { } } + for _, eventsFile := range sl.enhancedFiles { + if eventsFile != nil { + err := eventsFile.Close() + if err != nil { + log.Warningf("Failed closing enhanced events file: %v.", err) + } + } + } + // create a sentinel to signal completion signalFile := filepath.Join(sl.sessionDir, fmt.Sprintf("%v.completed", sl.SessionID.String())) err := ioutil.WriteFile(signalFile, []byte("completed"), 0640) @@ -211,8 +245,12 @@ func (sl *DiskSessionLogger) finalize() error { return nil } -// eventsFileName consists of session id and the first global event index recorded there -func eventsFileName(dataDir string, sessionID session.ID, eventIndex int64) string { +// eventsFileName consists of session id and the first global event index +// recorded. Optionally for enhanced session recording events, the event type. +func eventsFileName(dataDir string, sessionID session.ID, eventType string, eventIndex int64) string { + if eventType != "" { + return filepath.Join(dataDir, fmt.Sprintf("%v-%v.%v-%v", sessionID.String(), eventIndex, eventType, eventsSuffix)) + } return filepath.Join(dataDir, fmt.Sprintf("%v-%v.%v", sessionID.String(), eventIndex, eventsSuffix)) } @@ -228,10 +266,10 @@ func (sl *DiskSessionLogger) openEventsFile(eventIndex int64) error { sl.Warningf("Failed to close file: %v", trace.DebugReport(err)) } } - eventsFileName := eventsFileName(sl.sessionDir, sl.SessionID, eventIndex) + eventsFileName := eventsFileName(sl.sessionDir, sl.SessionID, "", eventIndex) // update the index file to write down that new events file has been created - data, err := json.Marshal(indexEntry{ + data, err := utils.FastMarshal(indexEntry{ FileName: filepath.Base(eventsFileName), Type: fileTypeEvents, Index: eventIndex, @@ -263,8 +301,8 @@ func (sl *DiskSessionLogger) openChunksFile(offset int64) error { } chunksFileName := chunksFileName(sl.sessionDir, sl.SessionID, offset) - // udpate the index file to write down that new chunks file has been created - data, err := json.Marshal(indexEntry{ + // Update the index file to write down that new chunks file has been created. + data, err := utils.FastMarshal(indexEntry{ FileName: filepath.Base(chunksFileName), Type: fileTypeChunks, Offset: offset, @@ -287,6 +325,56 @@ func (sl *DiskSessionLogger) openChunksFile(offset int64) error { return nil } +func (sl *DiskSessionLogger) openEnhancedFile(eventType string, eventIndex int64) error { + eventsFile, ok := sl.enhancedFiles[eventType] + if !ok { + return trace.BadParameter("unknown event type: %v", eventType) + } + + // If an events file is currently open, close it. + if eventsFile != nil { + err := eventsFile.Close() + if err != nil { + sl.Warningf("Failed to close file: %v.", trace.DebugReport(err)) + } + } + + // Create a new events file. + eventsFileName := eventsFileName(sl.sessionDir, sl.SessionID, eventType, eventIndex) + + // If the event is an enhanced event overwrite with the type of enhanced event. + var indexType string + switch eventType { + case SessionCommandEvent, SessionDiskEvent, SessionNetworkEvent: + indexType = eventType + default: + indexType = fileTypeEvents + } + + // Update the index file to write down that new events file has been created. + data, err := utils.FastMarshal(indexEntry{ + FileName: filepath.Base(eventsFileName), + Type: indexType, + Index: eventIndex, + }) + if err != nil { + return trace.Wrap(err) + } + _, err = fmt.Fprintf(sl.indexFile, "%v\n", string(data)) + if err != nil { + return trace.Wrap(err) + } + + // Open and store new file for writing events. + file, err := os.OpenFile(eventsFileName, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0640) + if err != nil { + return trace.Wrap(err) + } + sl.enhancedFiles[eventType] = newGzipWriter(file) + + return nil +} + // PostSessionSlice takes series of events associated with the session // and writes them to events files and data file for future replays func (sl *DiskSessionLogger) PostSessionSlice(slice SessionSlice) error { @@ -302,6 +390,19 @@ func (sl *DiskSessionLogger) PostSessionSlice(slice SessionSlice) error { return sl.flush() } +// PrintEventFromChunk returns a print event converted from session chunk. +func PrintEventFromChunk(chunk *SessionChunk) printEvent { + return printEvent{ + Start: time.Unix(0, chunk.Time).In(time.UTC).Round(time.Millisecond), + Type: SessionPrintEvent, + Bytes: int64(len(chunk.Data)), + DelayMilliseconds: chunk.Delay, + Offset: chunk.Offset, + EventIndex: chunk.EventIndex, + ChunkIndex: chunk.ChunkIndex, + } +} + // EventFromChunk returns event converted from session chunk func EventFromChunk(sessionID string, chunk *SessionChunk) (EventFields, error) { var fields EventFields @@ -320,59 +421,138 @@ func EventFromChunk(sessionID string, chunk *SessionChunk) (EventFields, error) return fields, nil } -func (sl *DiskSessionLogger) writeChunk(sessionID string, chunk *SessionChunk) (written int, err error) { +func (sl *DiskSessionLogger) writeChunk(sessionID string, chunk *SessionChunk) (int, error) { + switch chunk.EventType { + // Timing events for TTY playback go to both a chunks file (the raw bytes) as + // well as well as the events file (structured events). + case SessionPrintEvent: + // If the session are not being recorded, don't capture any print events. + if !sl.RecordSessions { + return len(chunk.Data), nil + } - // this section enforces the following invariant: - // a single events file only contains successive events + n, err := sl.writeEventChunk(sessionID, chunk) + if err != nil { + return n, trace.Wrap(err) + } + n, err = sl.writePrintChunk(sessionID, chunk) + if err != nil { + return n, trace.Wrap(err) + } + return n, nil + // Enhanced session recording events all go to their own events files. + case SessionCommandEvent, SessionDiskEvent, SessionNetworkEvent: + return sl.writeEnhancedChunk(sessionID, chunk) + // All other events get put into the general events file. These are events like + // session.join, session.end, etc. + default: + return sl.writeEventChunk(sessionID, chunk) + } +} + +func (sl *DiskSessionLogger) writeEventChunk(sessionID string, chunk *SessionChunk) (int, error) { + var bytes []byte + var err error + + // This section enforces the following invariant: a single events file only + // contains successive events. If means if an event arrives that is older or + // newer than the next expected event, a new file for that chunk is created. if sl.lastEventIndex == -1 || chunk.EventIndex-1 != sl.lastEventIndex { if err := sl.openEventsFile(chunk.EventIndex); err != nil { return -1, trace.Wrap(err) } } + + // Update index for the last event that was processed. sl.lastEventIndex = chunk.EventIndex - eventStart := time.Unix(0, chunk.Time).In(time.UTC).Round(time.Millisecond) - if chunk.EventType != SessionPrintEvent { + + // Marshal event. Note that print events are marshalled somewhat differently + // than all other events. + switch chunk.EventType { + case SessionPrintEvent: + bytes, err = utils.FastMarshal(PrintEventFromChunk(chunk)) + if err != nil { + return -1, trace.Wrap(err) + } + default: + // Convert to a marshallable event. fields, err := EventFromChunk(sessionID, chunk) if err != nil { return -1, trace.Wrap(err) } - data, err := json.Marshal(fields) + bytes, err = utils.FastMarshal(fields) if err != nil { return -1, trace.Wrap(err) } - return sl.eventsFile.Write(append(data, '\n')) } - if !sl.RecordSessions { - return len(chunk.Data), nil + + n, err := sl.eventsFile.Write(append(bytes, '\n')) + if err != nil { + return -1, trace.Wrap(err) } - // this section enforces the following invariant: - // a single chunks file only contains successive chunks + return n, nil +} + +func (sl *DiskSessionLogger) writePrintChunk(sessionID string, chunk *SessionChunk) (int, error) { + // This section enforces the following invariant: a single events file only + // contains successive events. If means if an event arrives that is older or + // newer than the next expected event, a new file for that chunk is created. if sl.lastChunkIndex == -1 || chunk.ChunkIndex-1 != sl.lastChunkIndex { if err := sl.openChunksFile(chunk.Offset); err != nil { return -1, trace.Wrap(err) } } + + // Update index for the last chunk that was processed. sl.lastChunkIndex = chunk.ChunkIndex - event := printEvent{ - Start: eventStart, - Type: SessionPrintEvent, - Bytes: int64(len(chunk.Data)), - DelayMilliseconds: chunk.Delay, - Offset: chunk.Offset, - EventIndex: chunk.EventIndex, - ChunkIndex: chunk.ChunkIndex, - } - bytes, err := json.Marshal(event) - if err != nil { - return -1, trace.Wrap(err) - } - _, err = sl.eventsFile.Write(append(bytes, '\n')) - if err != nil { - return -1, trace.Wrap(err) - } + return sl.chunksFile.Write(chunk.Data) } +func (sl *DiskSessionLogger) writeEnhancedChunk(sessionID string, chunk *SessionChunk) (int, error) { + var bytes []byte + var err error + + // Extract last index of particular event (command, disk, network). + lastIndex, ok := sl.enhancedIndexes[chunk.EventType] + if !ok { + return -1, trace.BadParameter("unknown event type: %v", chunk.EventType) + } + + // This section enforces the following invariant: a single events file only + // contains successive events. If means if an event arrives that is older or + // newer than the next expected event, a new file for that chunk is created. + if lastIndex == -1 || chunk.EventIndex-1 != lastIndex { + if err := sl.openEnhancedFile(chunk.EventType, chunk.EventIndex); err != nil { + return -1, trace.Wrap(err) + } + } + + // Update index for the last event that was processed. + sl.enhancedIndexes[chunk.EventType] = chunk.EventIndex + + // Convert to a marshallable event. + fields, err := EventFromChunk(sessionID, chunk) + if err != nil { + return -1, trace.Wrap(err) + } + bytes, err = utils.FastMarshal(fields) + if err != nil { + return -1, trace.Wrap(err) + } + + // Write event to appropriate file. + eventsFile, ok := sl.enhancedFiles[chunk.EventType] + if !ok { + return -1, trace.BadParameter("unknown event type: %v", chunk.EventType) + } + n, err := eventsFile.Write(append(bytes, '\n')) + if err != nil { + return -1, trace.Wrap(err) + } + return n, nil +} + func diff(before, after time.Time) int64 { d := int64(after.Sub(before) / time.Millisecond) if d < 0 { diff --git a/lib/service/cfg.go b/lib/service/cfg.go index df24a66cc25..666ec31cc01 100644 --- a/lib/service/cfg.go +++ b/lib/service/cfg.go @@ -28,6 +28,7 @@ import ( "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/backend/lite" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/limiter" @@ -172,6 +173,9 @@ type Config struct { // FIPS means FedRAMP/FIPS 140-2 compliant configuration was requested. FIPS bool + + // BPFConfig holds configuration for the BPF service. + BPFConfig *bpf.Config } // ApplyToken assigns a given token to all internal services but only if token @@ -414,6 +418,9 @@ type SSHConfig struct { // PublicAddrs affects the SSH host principals and DNS names added to the SSH and TLS certs. PublicAddrs []utils.NetAddr + + // BPF holds BPF configuration for Teleport. + BPF *bpf.Config } // MakeDefaultConfig creates a new Config structure and populates it with defaults @@ -447,7 +454,7 @@ func ApplyDefaults(cfg *Config) { log.Errorf("Failed to determine hostname: %v.", err) } - // global defaults + // Global defaults. cfg.Hostname = hostname cfg.DataDir = defaults.DataDir cfg.Console = os.Stdout @@ -456,7 +463,7 @@ func ApplyDefaults(cfg *Config) { cfg.KEXAlgorithms = kex cfg.MACAlgorithms = macs - // defaults for the auth service: + // Auth service defaults. cfg.Auth.Enabled = true cfg.Auth.SSHAddr = *defaults.AuthListenAddr() cfg.Auth.StorageConfig.Type = lite.GetName() @@ -470,22 +477,23 @@ func ApplyDefaults(cfg *Config) { cfg.Auth.Preference = ap cfg.Auth.LicenseFile = filepath.Join(cfg.DataDir, defaults.LicenseFile) - // defaults for the SSH proxy service: + // Proxy service defaults. cfg.Proxy.Enabled = true cfg.Proxy.SSHAddr = *defaults.ProxyListenAddr() cfg.Proxy.WebAddr = *defaults.ProxyWebListenAddr() cfg.Proxy.ReverseTunnelListenAddr = *defaults.ReverseTunnelListenAddr() defaults.ConfigureLimiter(&cfg.Proxy.Limiter) - // defaults for the Kubernetes proxy service + // Kubernetes proxy service defaults. cfg.Proxy.Kube.Enabled = false cfg.Proxy.Kube.ListenAddr = *defaults.KubeProxyListenAddr() - // defaults for the SSH service: + // SSH service defaults. cfg.SSH.Enabled = true cfg.SSH.Shell = defaults.DefaultShell defaults.ConfigureLimiter(&cfg.SSH.Limiter) cfg.SSH.PAM = &pam.Config{Enabled: false} + cfg.SSH.BPF = &bpf.Config{Enabled: false} } // ApplyFIPSDefaults updates default configuration to be FedRAMP/FIPS 140-2 diff --git a/lib/service/service.go b/lib/service/service.go index 2f462e5cca0..ee8753e141f 100644 --- a/lib/service/service.go +++ b/lib/service/service.go @@ -22,9 +22,6 @@ import ( "context" "crypto/tls" "fmt" - "github.com/gravitational/teleport/lib/backend/firestore" - "github.com/gravitational/teleport/lib/events/firestoreevents" - "github.com/gravitational/teleport/lib/events/gcssessions" "io" "io/ioutil" "net" @@ -46,14 +43,18 @@ import ( "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/backend/dynamo" "github.com/gravitational/teleport/lib/backend/etcdbk" + "github.com/gravitational/teleport/lib/backend/firestore" "github.com/gravitational/teleport/lib/backend/lite" "github.com/gravitational/teleport/lib/backend/memory" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/cache" "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/events/dynamoevents" "github.com/gravitational/teleport/lib/events/filesessions" + "github.com/gravitational/teleport/lib/events/firestoreevents" + "github.com/gravitational/teleport/lib/events/gcssessions" "github.com/gravitational/teleport/lib/events/s3sessions" kubeproxy "github.com/gravitational/teleport/lib/kube/proxy" "github.com/gravitational/teleport/lib/limiter" @@ -500,7 +501,10 @@ func waitAndReload(ctx context.Context, cfg Config, srv Process, newTeleport New // NewTeleport takes the daemon configuration, instantiates all required services // and starts them under a supervisor, returning the supervisor object. func NewTeleport(cfg *Config) (*TeleportProcess, error) { - // before we do anything reset the SIGINT handler back to the default + var err error + var ebpf bpf.BPF + + // Before we do anything reset the SIGINT handler back to the default. system.ResetInterruptSignalHandler() // If FIPS mode was requested make sure binary is build against BoringCrypto. @@ -516,8 +520,18 @@ func NewTeleport(cfg *Config) (*TeleportProcess, error) { return nil, trace.Wrap(err, "configuration error") } + // If this is a Teleport node and BPF is enabled, start BPF programs early + // in the startup process. This allows Teleport to fail right away if + // BPF-based auditing is configured but can not start for whatever reason. + if cfg.SSH.Enabled { + ebpf, err = bpf.New(cfg.SSH.BPF) + if err != nil { + return nil, trace.Wrap(err) + } + } + // create the data directory if it's missing - _, err := os.Stat(cfg.DataDir) + _, err = os.Stat(cfg.DataDir) if os.IsNotExist(err) { err := os.MkdirAll(cfg.DataDir, os.ModeDir|0700) if err != nil { @@ -646,7 +660,7 @@ func NewTeleport(cfg *Config) (*TeleportProcess, error) { } if cfg.SSH.Enabled { - if err := process.initSSH(); err != nil { + if err := process.initSSH(ebpf); err != nil { return nil, err } serviceStarted = true @@ -1368,7 +1382,7 @@ func (process *TeleportProcess) proxyPublicAddr() utils.NetAddr { } // initSSH initializes the "node" role, i.e. a simple SSH server connected to the auth server. -func (process *TeleportProcess) initSSH() error { +func (process *TeleportProcess) initSSH(ebpf bpf.BPF) error { process.registerWithAuthServer(teleport.RoleNode, SSHIdentityEvent) eventsC := make(chan Event) process.WaitForEvent(process.ExitContext(), SSHIdentityEvent, eventsC) @@ -1460,6 +1474,7 @@ func (process *TeleportProcess) initSSH() error { regular.SetRotationGetter(process.getRotation), regular.SetUseTunnel(conn.UseTunnel()), regular.SetFIPS(cfg.FIPS), + regular.SetBPF(ebpf), ) if err != nil { return trace.Wrap(err) @@ -1547,6 +1562,9 @@ func (process *TeleportProcess) initSSH() error { agentPool.Stop() } + // Close BPF service. + warnOnErr(ebpf.Close()) + log.Infof("Exited.") }) @@ -2452,4 +2470,4 @@ func initSelfSignedHTTPSCert(cfg *Config) (err error) { return trace.Wrap(err, "error writing key PEM") } return nil -} \ No newline at end of file +} diff --git a/lib/services/role.go b/lib/services/role.go index 60a201fc217..fd1b2c0f831 100644 --- a/lib/services/role.go +++ b/lib/services/role.go @@ -101,6 +101,7 @@ func NewAdminRole() Role { MaxSessionTTL: NewDuration(defaults.MaxCertDuration), PortForwarding: NewBoolOption(true), ForwardAgent: NewBool(true), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ Namespaces: []string{defaults.Namespace}, @@ -151,6 +152,7 @@ func RoleForUser(u User) Role { MaxSessionTTL: NewDuration(defaults.MaxCertDuration), PortForwarding: NewBoolOption(true), ForwardAgent: NewBool(true), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ Namespaces: []string{defaults.Namespace}, @@ -604,7 +606,7 @@ func (r *RoleV3) CheckAndSetDefaults() error { return trace.Wrap(err) } - // make sure we have defaults for all fields + // Make sure all fields have defaults. if r.Spec.Options.CertificateFormat == "" { r.Spec.Options.CertificateFormat = teleport.CertificateFormatStandard } @@ -614,6 +616,9 @@ func (r *RoleV3) CheckAndSetDefaults() error { if r.Spec.Options.PortForwarding == nil { r.Spec.Options.PortForwarding = NewBoolOption(true) } + if len(r.Spec.Options.BPF) == 0 { + r.Spec.Options.BPF = defaults.EnhancedEvents() + } if r.Spec.Allow.Namespaces == nil { r.Spec.Allow.Namespaces = []string{defaults.Namespace} } @@ -624,6 +629,16 @@ func (r *RoleV3) CheckAndSetDefaults() error { r.Spec.Deny.Namespaces = []string{defaults.Namespace} } + // Validate that enhanced recording options are all valid. + for _, opt := range r.Spec.Options.BPF { + if opt == teleport.EnhancedRecordingCommand || + opt == teleport.EnhancedRecordingDisk || + opt == teleport.EnhancedRecordingNetwork { + continue + } + return trace.BadParameter("found invalid option in session_recording: %v", opt) + } + // if we find {{ or }} but the syntax is invalid, the role is invalid for _, condition := range []RoleConditionType{Allow, Deny} { for _, login := range r.GetLogins(condition) { @@ -680,7 +695,8 @@ func (o RoleOptions) Equals(other RoleOptions) bool { BoolDefaultTrue(o.PortForwarding) == BoolDefaultTrue(other.PortForwarding) && o.CertificateFormat == other.CertificateFormat && o.ClientIdleTimeout.Value() == other.ClientIdleTimeout.Value() && - o.DisconnectExpiredCert.Value() == other.DisconnectExpiredCert.Value()) + o.DisconnectExpiredCert.Value() == other.DisconnectExpiredCert.Value() && + utils.StringSlicesEqual(o.BPF, other.BPF)) } // Equals returns true if the role conditions (logins, namespaces, labels, @@ -1180,6 +1196,7 @@ func (r *RoleV2) V3() *RoleV3 { CertificateFormat: teleport.CertificateFormatStandard, MaxSessionTTL: r.GetMaxSessionTTL(), PortForwarding: NewBoolOption(true), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ Logins: r.GetLogins(), @@ -1295,6 +1312,10 @@ type AccessChecker interface { // CertificateFormat returns the most permissive certificate format in a // RoleSet. CertificateFormat() string + + // EnhancedRecordingSet returns a set of events that will be recorded + // for enhanced session recording. + EnhancedRecordingSet() map[string]bool } // FromSpec returns new RoleSet created from spec @@ -1763,6 +1784,21 @@ func (set RoleSet) CertificateFormat() string { return formats[0] } +// EnhancedRecordingSet returns the set of enhanced session recording +// events to capture for thi role set. +func (set RoleSet) EnhancedRecordingSet() map[string]bool { + m := make(map[string]bool) + + // Loop over all roles and create a set of all options. + for _, role := range set { + for _, opt := range role.GetOptions().BPF { + m[opt] = true + } + } + + return m +} + // certificatePriority returns the priority of the certificate format. The // most permissive has lowest value. func certificatePriority(s string) int { @@ -2193,7 +2229,11 @@ const RoleSpecV3SchemaTemplate = `{ "port_forwarding": { "type": ["boolean", "string"] }, "cert_format": { "type": "string" }, "client_idle_timeout": { "type": "string" }, - "disconnect_expired_cert": { "type": ["boolean", "string"] } + "disconnect_expired_cert": { "type": ["boolean", "string"] }, + "enhanced_recording": { + "type": "array", + "items": { "type": "string" } + } } }, "allow": { "$ref": "#/definitions/role_condition" }, diff --git a/lib/services/role_test.go b/lib/services/role_test.go index d1108daf61c..1da4ccd862c 100644 --- a/lib/services/role_test.go +++ b/lib/services/role_test.go @@ -94,94 +94,95 @@ func (s *RoleSuite) TestRoleParse(c *C) { role: RoleV3{}, error: trace.BadParameter("failed to validate: name: name is required"), }, + { name: "validation error, missing resources", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "allow": { - "node_labels": {"a": "b"}, - "namespaces": ["default"], - "rules": [ - { - "verbs": ["read", "list"] - } - ] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "allow": { + "node_labels": {"a": "b"}, + "namespaces": ["default"], + "rules": [ + { + "verbs": ["read", "list"] + } + ] + } + } + }`, error: trace.BadParameter(""), matchMessage: ".*missing resources.*", }, { name: "validation error, missing verbs", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "allow": { - "node_labels": {"a": "b"}, - "namespaces": ["default"], - "rules": [ - { - "resources": ["role"] - } - ] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "allow": { + "node_labels": {"a": "b"}, + "namespaces": ["default"], + "rules": [ + { + "resources": ["role"] + } + ] + } + } + }`, error: trace.BadParameter(""), matchMessage: ".*missing verbs.*", }, { name: "validation error, unsupported function in where", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "allow": { - "node_labels": {"a": "b"}, - "namespaces": ["default"], - "rules": [ - { - "resources": ["role"], - "verbs": ["read", "list"], - "where": "containz(user.spec.traits[\"groups\"], \"prod\")" - } - ] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "allow": { + "node_labels": {"a": "b"}, + "namespaces": ["default"], + "rules": [ + { + "resources": ["role"], + "verbs": ["read", "list"], + "where": "containz(user.spec.traits[\"groups\"], \"prod\")" + } + ] + } + } + }`, error: trace.BadParameter(""), matchMessage: ".*unsupported function: containz.*", }, { name: "validation error, unsupported function in actions", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "allow": { - "node_labels": {"a": "b"}, - "namespaces": ["default"], - "rules": [ - { - "resources": ["role"], - "verbs": ["read", "list"], - "where": "contains(user.spec.traits[\"groups\"], \"prod\")", - "actions": [ - "zzz(\"info\", \"log entry\")" - ] - } - ] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "allow": { + "node_labels": {"a": "b"}, + "namespaces": ["default"], + "rules": [ + { + "resources": ["role"], + "verbs": ["read", "list"], + "where": "contains(user.spec.traits[\"groups\"], \"prod\")", + "actions": [ + "zzz(\"info\", \"log entry\")" + ] + } + ] + } + } + }`, error: trace.BadParameter(""), matchMessage: ".*unsupported function: zzz.*", }, @@ -200,6 +201,7 @@ func (s *RoleSuite) TestRoleParse(c *C) { CertificateFormat: teleport.CertificateFormatStandard, MaxSessionTTL: NewDuration(defaults.MaxCertDuration), PortForwarding: NewBoolOption(true), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ NodeLabels: Labels{Wildcard: []string{Wildcard}}, @@ -215,36 +217,37 @@ func (s *RoleSuite) TestRoleParse(c *C) { { name: "full valid role", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1", "labels": {"a-b": "c"}}, - "spec": { - "options": { - "cert_format": "standard", - "max_session_ttl": "20h", - "port_forwarding": true, - "client_idle_timeout": "17m", - "disconnect_expired_cert": "yes" - }, - "allow": { - "node_labels": {"a": "b", "c-d": "e"}, - "namespaces": ["default"], - "rules": [ - { - "resources": ["role"], - "verbs": ["read", "list"], - "where": "contains(user.spec.traits[\"groups\"], \"prod\")", - "actions": [ - "log(\"info\", \"log entry\")" - ] - } - ] - }, - "deny": { - "logins": ["c"] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1", "labels": {"a-b": "c"}}, + "spec": { + "options": { + "cert_format": "standard", + "max_session_ttl": "20h", + "port_forwarding": true, + "client_idle_timeout": "17m", + "disconnect_expired_cert": "yes", + "enhanced_recording": ["command", "network"] + }, + "allow": { + "node_labels": {"a": "b", "c-d": "e"}, + "namespaces": ["default"], + "rules": [ + { + "resources": ["role"], + "verbs": ["read", "list"], + "where": "contains(user.spec.traits[\"groups\"], \"prod\")", + "actions": [ + "log(\"info\", \"log entry\")" + ] + } + ] + }, + "deny": { + "logins": ["c"] + } + } + }`, role: RoleV3{ Kind: KindRole, Version: V3, @@ -260,6 +263,7 @@ func (s *RoleSuite) TestRoleParse(c *C) { PortForwarding: NewBoolOption(true), ClientIdleTimeout: NewDuration(17 * time.Minute), DisconnectExpiredCert: NewBool(true), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ NodeLabels: Labels{"a": []string{"b"}, "c-d": []string{"e"}}, @@ -286,37 +290,38 @@ func (s *RoleSuite) TestRoleParse(c *C) { { name: "alternative options form", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "options": { - "cert_format": "standard", - "max_session_ttl": "20h", - "port_forwarding": "yes", - "forward_agent": "yes", - "client_idle_timeout": "never", - "disconnect_expired_cert": "no" - }, - "allow": { - "node_labels": {"a": "b"}, - "namespaces": ["default"], - "rules": [ - { - "resources": ["role"], - "verbs": ["read", "list"], - "where": "contains(user.spec.traits[\"groups\"], \"prod\")", - "actions": [ - "log(\"info\", \"log entry\")" - ] - } - ] - }, - "deny": { - "logins": ["c"] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "options": { + "cert_format": "standard", + "max_session_ttl": "20h", + "port_forwarding": "yes", + "forward_agent": "yes", + "client_idle_timeout": "never", + "disconnect_expired_cert": "no", + "enhanced_recording": ["command", "network"] + }, + "allow": { + "node_labels": {"a": "b"}, + "namespaces": ["default"], + "rules": [ + { + "resources": ["role"], + "verbs": ["read", "list"], + "where": "contains(user.spec.traits[\"groups\"], \"prod\")", + "actions": [ + "log(\"info\", \"log entry\")" + ] + } + ] + }, + "deny": { + "logins": ["c"] + } + } + }`, role: RoleV3{ Kind: KindRole, Version: V3, @@ -332,6 +337,7 @@ func (s *RoleSuite) TestRoleParse(c *C) { PortForwarding: NewBoolOption(true), ClientIdleTimeout: NewDuration(0), DisconnectExpiredCert: NewBool(false), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ NodeLabels: Labels{"a": []string{"b"}}, @@ -358,26 +364,27 @@ func (s *RoleSuite) TestRoleParse(c *C) { { name: "non-scalar and scalar values of labels", in: `{ - "kind": "role", - "version": "v3", - "metadata": {"name": "name1"}, - "spec": { - "options": { - "cert_format": "standard", - "max_session_ttl": "20h", - "port_forwarding": "yes", - "forward_agent": "yes", - "client_idle_timeout": "never", - "disconnect_expired_cert": "no" - }, - "allow": { - "node_labels": {"a": "b", "key": ["val"], "key2": ["val2", "val3"]} - }, - "deny": { - "logins": ["c"] - } - } - }`, + "kind": "role", + "version": "v3", + "metadata": {"name": "name1"}, + "spec": { + "options": { + "cert_format": "standard", + "max_session_ttl": "20h", + "port_forwarding": "yes", + "forward_agent": "yes", + "client_idle_timeout": "never", + "disconnect_expired_cert": "no", + "enhanced_recording": ["command", "network"] + }, + "allow": { + "node_labels": {"a": "b", "key": ["val"], "key2": ["val2", "val3"]} + }, + "deny": { + "logins": ["c"] + } + } + }`, role: RoleV3{ Kind: KindRole, Version: V3, @@ -393,6 +400,7 @@ func (s *RoleSuite) TestRoleParse(c *C) { PortForwarding: NewBoolOption(true), ClientIdleTimeout: NewDuration(0), DisconnectExpiredCert: NewBool(false), + BPF: defaults.EnhancedEvents(), }, Allow: RoleConditions{ NodeLabels: Labels{ diff --git a/lib/services/suite/suite.go b/lib/services/suite/suite.go index 748526f36ac..0e7e8caf2aa 100644 --- a/lib/services/suite/suite.go +++ b/lib/services/suite/suite.go @@ -555,6 +555,7 @@ func (s *ServicesTestSuite) RolesCRUD(c *check.C) { MaxSessionTTL: services.Duration(time.Hour), PortForwarding: services.NewBoolOption(true), CertificateFormat: teleport.CertificateFormatStandard, + BPF: defaults.EnhancedEvents(), }, Allow: services.RoleConditions{ Logins: []string{"root", "bob"}, diff --git a/lib/services/types.pb.go b/lib/services/types.pb.go index 1398a1699fd..5e315c096d4 100644 --- a/lib/services/types.pb.go +++ b/lib/services/types.pb.go @@ -57,7 +57,7 @@ func (x RequestState) String() string { return proto.EnumName(RequestState_name, int32(x)) } func (RequestState) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{0} + return fileDescriptor_types_e2396f0bdbc60acd, []int{0} } type KeepAlive struct { @@ -78,7 +78,7 @@ func (m *KeepAlive) Reset() { *m = KeepAlive{} } func (m *KeepAlive) String() string { return proto.CompactTextString(m) } func (*KeepAlive) ProtoMessage() {} func (*KeepAlive) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{0} + return fileDescriptor_types_e2396f0bdbc60acd, []int{0} } func (m *KeepAlive) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -131,7 +131,7 @@ func (m *Metadata) Reset() { *m = Metadata{} } func (m *Metadata) String() string { return proto.CompactTextString(m) } func (*Metadata) ProtoMessage() {} func (*Metadata) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{1} + return fileDescriptor_types_e2396f0bdbc60acd, []int{1} } func (m *Metadata) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -190,7 +190,7 @@ type Rotation struct { func (m *Rotation) Reset() { *m = Rotation{} } func (*Rotation) ProtoMessage() {} func (*Rotation) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{2} + return fileDescriptor_types_e2396f0bdbc60acd, []int{2} } func (m *Rotation) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -237,7 +237,7 @@ func (m *RotationSchedule) Reset() { *m = RotationSchedule{} } func (m *RotationSchedule) String() string { return proto.CompactTextString(m) } func (*RotationSchedule) ProtoMessage() {} func (*RotationSchedule) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{3} + return fileDescriptor_types_e2396f0bdbc60acd, []int{3} } func (m *RotationSchedule) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -286,7 +286,7 @@ func (m *ResourceHeader) Reset() { *m = ResourceHeader{} } func (m *ResourceHeader) String() string { return proto.CompactTextString(m) } func (*ResourceHeader) ProtoMessage() {} func (*ResourceHeader) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{4} + return fileDescriptor_types_e2396f0bdbc60acd, []int{4} } func (m *ResourceHeader) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -335,7 +335,7 @@ type ServerV2 struct { func (m *ServerV2) Reset() { *m = ServerV2{} } func (*ServerV2) ProtoMessage() {} func (*ServerV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{5} + return fileDescriptor_types_e2396f0bdbc60acd, []int{5} } func (m *ServerV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -388,7 +388,7 @@ func (m *ServerSpecV2) Reset() { *m = ServerSpecV2{} } func (m *ServerSpecV2) String() string { return proto.CompactTextString(m) } func (*ServerSpecV2) ProtoMessage() {} func (*ServerSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{6} + return fileDescriptor_types_e2396f0bdbc60acd, []int{6} } func (m *ServerSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -435,7 +435,7 @@ func (m *CommandLabelV2) Reset() { *m = CommandLabelV2{} } func (m *CommandLabelV2) String() string { return proto.CompactTextString(m) } func (*CommandLabelV2) ProtoMessage() {} func (*CommandLabelV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{7} + return fileDescriptor_types_e2396f0bdbc60acd, []int{7} } func (m *CommandLabelV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -479,7 +479,7 @@ func (m *TLSKeyPair) Reset() { *m = TLSKeyPair{} } func (m *TLSKeyPair) String() string { return proto.CompactTextString(m) } func (*TLSKeyPair) ProtoMessage() {} func (*TLSKeyPair) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{8} + return fileDescriptor_types_e2396f0bdbc60acd, []int{8} } func (m *TLSKeyPair) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -528,7 +528,7 @@ type CertAuthorityV2 struct { func (m *CertAuthorityV2) Reset() { *m = CertAuthorityV2{} } func (*CertAuthorityV2) ProtoMessage() {} func (*CertAuthorityV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{9} + return fileDescriptor_types_e2396f0bdbc60acd, []int{9} } func (m *CertAuthorityV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -591,7 +591,7 @@ func (m *CertAuthoritySpecV2) Reset() { *m = CertAuthoritySpecV2{} } func (m *CertAuthoritySpecV2) String() string { return proto.CompactTextString(m) } func (*CertAuthoritySpecV2) ProtoMessage() {} func (*CertAuthoritySpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{10} + return fileDescriptor_types_e2396f0bdbc60acd, []int{10} } func (m *CertAuthoritySpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -636,7 +636,7 @@ func (m *RoleMapping) Reset() { *m = RoleMapping{} } func (m *RoleMapping) String() string { return proto.CompactTextString(m) } func (*RoleMapping) ProtoMessage() {} func (*RoleMapping) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{11} + return fileDescriptor_types_e2396f0bdbc60acd, []int{11} } func (m *RoleMapping) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -683,7 +683,7 @@ type ProvisionTokenV1 struct { func (m *ProvisionTokenV1) Reset() { *m = ProvisionTokenV1{} } func (*ProvisionTokenV1) ProtoMessage() {} func (*ProvisionTokenV1) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{12} + return fileDescriptor_types_e2396f0bdbc60acd, []int{12} } func (m *ProvisionTokenV1) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -732,7 +732,7 @@ type ProvisionTokenV2 struct { func (m *ProvisionTokenV2) Reset() { *m = ProvisionTokenV2{} } func (*ProvisionTokenV2) ProtoMessage() {} func (*ProvisionTokenV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{13} + return fileDescriptor_types_e2396f0bdbc60acd, []int{13} } func (m *ProvisionTokenV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -776,7 +776,7 @@ func (m *ProvisionTokenSpecV2) Reset() { *m = ProvisionTokenSpecV2{} } func (m *ProvisionTokenSpecV2) String() string { return proto.CompactTextString(m) } func (*ProvisionTokenSpecV2) ProtoMessage() {} func (*ProvisionTokenSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{14} + return fileDescriptor_types_e2396f0bdbc60acd, []int{14} } func (m *ProvisionTokenSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -825,7 +825,7 @@ type StaticTokensV2 struct { func (m *StaticTokensV2) Reset() { *m = StaticTokensV2{} } func (*StaticTokensV2) ProtoMessage() {} func (*StaticTokensV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{15} + return fileDescriptor_types_e2396f0bdbc60acd, []int{15} } func (m *StaticTokensV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -868,7 +868,7 @@ func (m *StaticTokensSpecV2) Reset() { *m = StaticTokensSpecV2{} } func (m *StaticTokensSpecV2) String() string { return proto.CompactTextString(m) } func (*StaticTokensSpecV2) ProtoMessage() {} func (*StaticTokensSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{16} + return fileDescriptor_types_e2396f0bdbc60acd, []int{16} } func (m *StaticTokensSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -917,7 +917,7 @@ type ClusterNameV2 struct { func (m *ClusterNameV2) Reset() { *m = ClusterNameV2{} } func (*ClusterNameV2) ProtoMessage() {} func (*ClusterNameV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{17} + return fileDescriptor_types_e2396f0bdbc60acd, []int{17} } func (m *ClusterNameV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -960,7 +960,7 @@ func (m *ClusterNameSpecV2) Reset() { *m = ClusterNameSpecV2{} } func (m *ClusterNameSpecV2) String() string { return proto.CompactTextString(m) } func (*ClusterNameSpecV2) ProtoMessage() {} func (*ClusterNameSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{18} + return fileDescriptor_types_e2396f0bdbc60acd, []int{18} } func (m *ClusterNameSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1009,7 +1009,7 @@ type ClusterConfigV3 struct { func (m *ClusterConfigV3) Reset() { *m = ClusterConfigV3{} } func (*ClusterConfigV3) ProtoMessage() {} func (*ClusterConfigV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{19} + return fileDescriptor_types_e2396f0bdbc60acd, []int{19} } func (m *ClusterConfigV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1072,7 +1072,7 @@ func (m *ClusterConfigSpecV3) Reset() { *m = ClusterConfigSpecV3{} } func (m *ClusterConfigSpecV3) String() string { return proto.CompactTextString(m) } func (*ClusterConfigSpecV3) ProtoMessage() {} func (*ClusterConfigSpecV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{20} + return fileDescriptor_types_e2396f0bdbc60acd, []int{20} } func (m *ClusterConfigSpecV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1125,7 +1125,7 @@ func (m *AuditConfig) Reset() { *m = AuditConfig{} } func (m *AuditConfig) String() string { return proto.CompactTextString(m) } func (*AuditConfig) ProtoMessage() {} func (*AuditConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{21} + return fileDescriptor_types_e2396f0bdbc60acd, []int{21} } func (m *AuditConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1175,7 +1175,7 @@ func (m *Namespace) Reset() { *m = Namespace{} } func (m *Namespace) String() string { return proto.CompactTextString(m) } func (*Namespace) ProtoMessage() {} func (*Namespace) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{22} + return fileDescriptor_types_e2396f0bdbc60acd, []int{22} } func (m *Namespace) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1215,7 +1215,7 @@ func (m *NamespaceSpec) Reset() { *m = NamespaceSpec{} } func (m *NamespaceSpec) String() string { return proto.CompactTextString(m) } func (*NamespaceSpec) ProtoMessage() {} func (*NamespaceSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{23} + return fileDescriptor_types_e2396f0bdbc60acd, []int{23} } func (m *NamespaceSpec) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1264,7 +1264,7 @@ type AccessRequestV3 struct { func (m *AccessRequestV3) Reset() { *m = AccessRequestV3{} } func (*AccessRequestV3) ProtoMessage() {} func (*AccessRequestV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{24} + return fileDescriptor_types_e2396f0bdbc60acd, []int{24} } func (m *AccessRequestV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1314,7 +1314,7 @@ func (m *AccessRequestSpecV3) Reset() { *m = AccessRequestSpecV3{} } func (m *AccessRequestSpecV3) String() string { return proto.CompactTextString(m) } func (*AccessRequestSpecV3) ProtoMessage() {} func (*AccessRequestSpecV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{25} + return fileDescriptor_types_e2396f0bdbc60acd, []int{25} } func (m *AccessRequestSpecV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1360,7 +1360,7 @@ func (m *AccessRequestFilter) Reset() { *m = AccessRequestFilter{} } func (m *AccessRequestFilter) String() string { return proto.CompactTextString(m) } func (*AccessRequestFilter) ProtoMessage() {} func (*AccessRequestFilter) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{26} + return fileDescriptor_types_e2396f0bdbc60acd, []int{26} } func (m *AccessRequestFilter) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1409,7 +1409,7 @@ type RoleV3 struct { func (m *RoleV3) Reset() { *m = RoleV3{} } func (*RoleV3) ProtoMessage() {} func (*RoleV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{27} + return fileDescriptor_types_e2396f0bdbc60acd, []int{27} } func (m *RoleV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1455,7 +1455,7 @@ func (m *RoleSpecV3) Reset() { *m = RoleSpecV3{} } func (m *RoleSpecV3) String() string { return proto.CompactTextString(m) } func (*RoleSpecV3) ProtoMessage() {} func (*RoleSpecV3) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{28} + return fileDescriptor_types_e2396f0bdbc60acd, []int{28} } func (m *RoleSpecV3) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1502,17 +1502,19 @@ type RoleOptions struct { // duration. ClientIdleTimeout Duration `protobuf:"varint,5,opt,name=ClientIdleTimeout,proto3,casttype=Duration" json:"client_idle_timeout,omitempty"` // DisconnectExpiredCert sets disconnect clients on expired certificates. - DisconnectExpiredCert Bool `protobuf:"varint,6,opt,name=DisconnectExpiredCert,proto3,casttype=Bool" json:"disconnect_expired_cert,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + DisconnectExpiredCert Bool `protobuf:"varint,6,opt,name=DisconnectExpiredCert,proto3,casttype=Bool" json:"disconnect_expired_cert,omitempty"` + // BPF defines what events to record for the BPF-based session recorder. + BPF []string `protobuf:"bytes,7,rep,name=BPF" json:"enhanced_recording,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *RoleOptions) Reset() { *m = RoleOptions{} } func (m *RoleOptions) String() string { return proto.CompactTextString(m) } func (*RoleOptions) ProtoMessage() {} func (*RoleOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{29} + return fileDescriptor_types_e2396f0bdbc60acd, []int{29} } func (m *RoleOptions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1566,7 +1568,7 @@ func (m *RoleConditions) Reset() { *m = RoleConditions{} } func (m *RoleConditions) String() string { return proto.CompactTextString(m) } func (*RoleConditions) ProtoMessage() {} func (*RoleConditions) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{30} + return fileDescriptor_types_e2396f0bdbc60acd, []int{30} } func (m *RoleConditions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1608,7 +1610,7 @@ func (m *AccessRequestConditions) Reset() { *m = AccessRequestConditions func (m *AccessRequestConditions) String() string { return proto.CompactTextString(m) } func (*AccessRequestConditions) ProtoMessage() {} func (*AccessRequestConditions) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{31} + return fileDescriptor_types_e2396f0bdbc60acd, []int{31} } func (m *AccessRequestConditions) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1657,7 +1659,7 @@ func (m *Rule) Reset() { *m = Rule{} } func (m *Rule) String() string { return proto.CompactTextString(m) } func (*Rule) ProtoMessage() {} func (*Rule) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{32} + return fileDescriptor_types_e2396f0bdbc60acd, []int{32} } func (m *Rule) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1699,7 +1701,7 @@ func (m *BoolValue) Reset() { *m = BoolValue{} } func (m *BoolValue) String() string { return proto.CompactTextString(m) } func (*BoolValue) ProtoMessage() {} func (*BoolValue) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{33} + return fileDescriptor_types_e2396f0bdbc60acd, []int{33} } func (m *BoolValue) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1748,7 +1750,7 @@ type UserV2 struct { func (m *UserV2) Reset() { *m = UserV2{} } func (*UserV2) ProtoMessage() {} func (*UserV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{34} + return fileDescriptor_types_e2396f0bdbc60acd, []int{34} } func (m *UserV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1811,7 +1813,7 @@ func (m *UserSpecV2) Reset() { *m = UserSpecV2{} } func (m *UserSpecV2) String() string { return proto.CompactTextString(m) } func (*UserSpecV2) ProtoMessage() {} func (*UserSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{35} + return fileDescriptor_types_e2396f0bdbc60acd, []int{35} } func (m *UserSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1856,7 +1858,7 @@ type ExternalIdentity struct { func (m *ExternalIdentity) Reset() { *m = ExternalIdentity{} } func (*ExternalIdentity) ProtoMessage() {} func (*ExternalIdentity) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{36} + return fileDescriptor_types_e2396f0bdbc60acd, []int{36} } func (m *ExternalIdentity) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1904,7 +1906,7 @@ func (m *LoginStatus) Reset() { *m = LoginStatus{} } func (m *LoginStatus) String() string { return proto.CompactTextString(m) } func (*LoginStatus) ProtoMessage() {} func (*LoginStatus) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{37} + return fileDescriptor_types_e2396f0bdbc60acd, []int{37} } func (m *LoginStatus) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1949,7 +1951,7 @@ type CreatedBy struct { func (m *CreatedBy) Reset() { *m = CreatedBy{} } func (*CreatedBy) ProtoMessage() {} func (*CreatedBy) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{38} + return fileDescriptor_types_e2396f0bdbc60acd, []int{38} } func (m *CreatedBy) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1995,7 +1997,7 @@ func (m *U2FRegistrationData) Reset() { *m = U2FRegistrationData{} } func (m *U2FRegistrationData) String() string { return proto.CompactTextString(m) } func (*U2FRegistrationData) ProtoMessage() {} func (*U2FRegistrationData) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{39} + return fileDescriptor_types_e2396f0bdbc60acd, []int{39} } func (m *U2FRegistrationData) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2043,7 +2045,7 @@ func (m *LocalAuthSecrets) Reset() { *m = LocalAuthSecrets{} } func (m *LocalAuthSecrets) String() string { return proto.CompactTextString(m) } func (*LocalAuthSecrets) ProtoMessage() {} func (*LocalAuthSecrets) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{40} + return fileDescriptor_types_e2396f0bdbc60acd, []int{40} } func (m *LocalAuthSecrets) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2089,7 +2091,7 @@ func (m *ConnectorRef) Reset() { *m = ConnectorRef{} } func (m *ConnectorRef) String() string { return proto.CompactTextString(m) } func (*ConnectorRef) ProtoMessage() {} func (*ConnectorRef) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{41} + return fileDescriptor_types_e2396f0bdbc60acd, []int{41} } func (m *ConnectorRef) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2131,7 +2133,7 @@ func (m *UserRef) Reset() { *m = UserRef{} } func (m *UserRef) String() string { return proto.CompactTextString(m) } func (*UserRef) ProtoMessage() {} func (*UserRef) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{42} + return fileDescriptor_types_e2396f0bdbc60acd, []int{42} } func (m *UserRef) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2181,7 +2183,7 @@ func (m *ReverseTunnelV2) Reset() { *m = ReverseTunnelV2{} } func (m *ReverseTunnelV2) String() string { return proto.CompactTextString(m) } func (*ReverseTunnelV2) ProtoMessage() {} func (*ReverseTunnelV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{43} + return fileDescriptor_types_e2396f0bdbc60acd, []int{43} } func (m *ReverseTunnelV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2228,7 +2230,7 @@ func (m *ReverseTunnelSpecV2) Reset() { *m = ReverseTunnelSpecV2{} } func (m *ReverseTunnelSpecV2) String() string { return proto.CompactTextString(m) } func (*ReverseTunnelSpecV2) ProtoMessage() {} func (*ReverseTunnelSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{44} + return fileDescriptor_types_e2396f0bdbc60acd, []int{44} } func (m *ReverseTunnelSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2277,7 +2279,7 @@ type TunnelConnectionV2 struct { func (m *TunnelConnectionV2) Reset() { *m = TunnelConnectionV2{} } func (*TunnelConnectionV2) ProtoMessage() {} func (*TunnelConnectionV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{45} + return fileDescriptor_types_e2396f0bdbc60acd, []int{45} } func (m *TunnelConnectionV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2325,7 +2327,7 @@ func (m *TunnelConnectionSpecV2) Reset() { *m = TunnelConnectionSpecV2{} func (m *TunnelConnectionSpecV2) String() string { return proto.CompactTextString(m) } func (*TunnelConnectionSpecV2) ProtoMessage() {} func (*TunnelConnectionSpecV2) Descriptor() ([]byte, []int) { - return fileDescriptor_types_1f32999e8058ec6f, []int{46} + return fileDescriptor_types_e2396f0bdbc60acd, []int{46} } func (m *TunnelConnectionSpecV2) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3996,6 +3998,21 @@ func (m *RoleOptions) MarshalTo(dAtA []byte) (int, error) { } i++ } + if len(m.BPF) > 0 { + for _, s := range m.BPF { + dAtA[i] = 0x3a + i++ + l = len(s) + for l >= 1<<7 { + dAtA[i] = uint8(uint64(l)&0x7f | 0x80) + l >>= 7 + i++ + } + dAtA[i] = uint8(l) + i++ + i += copy(dAtA[i:], s) + } + } if m.XXX_unrecognized != nil { i += copy(dAtA[i:], m.XXX_unrecognized) } @@ -5627,6 +5644,12 @@ func (m *RoleOptions) Size() (n int) { if m.DisconnectExpiredCert { n += 2 } + if len(m.BPF) > 0 { + for _, s := range m.BPF { + l = len(s) + n += 1 + l + sovTypes(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -11347,6 +11370,35 @@ func (m *RoleOptions) Unmarshal(dAtA []byte) error { } } m.DisconnectExpiredCert = Bool(v != 0) + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BPF", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowTypes + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthTypes + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BPF = append(m.BPF, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipTypes(dAtA[iNdEx:]) @@ -14188,239 +14240,241 @@ var ( ErrIntOverflowTypes = fmt.Errorf("proto: integer overflow") ) -func init() { proto.RegisterFile("types.proto", fileDescriptor_types_1f32999e8058ec6f) } +func init() { proto.RegisterFile("types.proto", fileDescriptor_types_e2396f0bdbc60acd) } -var fileDescriptor_types_1f32999e8058ec6f = []byte{ - // 3693 bytes of a gzipped FileDescriptorProto +var fileDescriptor_types_e2396f0bdbc60acd = []byte{ + // 3717 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xdc, 0x3a, 0x4d, 0x73, 0x1c, 0xc7, 0x75, 0xdc, 0x0f, 0x00, 0xbb, 0x6f, 0x01, 0x10, 0x6c, 0x80, 0xe4, 0xf2, 0x43, 0x1c, 0x68, 0x14, - 0xc9, 0x94, 0x43, 0x03, 0x31, 0x18, 0x29, 0x96, 0x62, 0x46, 0xc2, 0x62, 0x41, 0x12, 0x06, 0x48, + 0xc9, 0x94, 0x43, 0x03, 0x31, 0x18, 0x29, 0x96, 0x62, 0x46, 0xc2, 0x62, 0x01, 0x12, 0x06, 0x48, 0x42, 0x03, 0x10, 0x2e, 0x57, 0x3e, 0x26, 0x83, 0x99, 0xc6, 0x62, 0x0a, 0xb3, 0x33, 0x9b, 0x99, - 0x1e, 0x90, 0x7b, 0x73, 0x25, 0xa9, 0x4a, 0x25, 0xae, 0xb2, 0x9d, 0x4a, 0xb9, 0xa2, 0xaa, 0xe4, - 0x90, 0x6b, 0x0e, 0xc9, 0x35, 0xa7, 0x1c, 0x52, 0x95, 0x03, 0x8f, 0x3e, 0xbb, 0x9c, 0x71, 0xa2, + 0x1e, 0x90, 0x7b, 0x73, 0x25, 0xa9, 0x4a, 0x25, 0xae, 0xb2, 0x93, 0x4a, 0xb9, 0xa2, 0xaa, 0xe4, + 0x90, 0x6b, 0x0e, 0xc9, 0x35, 0xa7, 0x1c, 0x52, 0xe5, 0x03, 0x8f, 0x3e, 0xbb, 0x9c, 0x71, 0xa2, 0x1c, 0x5c, 0xb5, 0x3f, 0x41, 0x97, 0xa4, 0xfa, 0x75, 0xcf, 0x4c, 0xcf, 0xee, 0x82, 0x58, 0xc9, - 0xba, 0x40, 0xa7, 0x9d, 0x7d, 0x5f, 0xdd, 0xef, 0xf5, 0xeb, 0xf7, 0x5e, 0xbf, 0x6e, 0x68, 0xb0, - 0x7e, 0x8f, 0x46, 0x2b, 0xbd, 0x30, 0x60, 0x01, 0xa9, 0x45, 0x34, 0x3c, 0x75, 0x6d, 0x1a, 0xdd, + 0xba, 0x50, 0xa7, 0x9d, 0x7d, 0x5f, 0xdd, 0xef, 0xf5, 0xeb, 0xf7, 0x5e, 0xbf, 0x6e, 0x68, 0xb0, + 0x7e, 0x8f, 0x46, 0x2b, 0xbd, 0x30, 0x60, 0x01, 0xa9, 0x45, 0x34, 0x3c, 0x73, 0x6d, 0x1a, 0xdd, 0x5c, 0xea, 0x04, 0x9d, 0x00, 0x81, 0xab, 0xfc, 0x4b, 0xe0, 0x6f, 0x6a, 0x9d, 0x20, 0xe8, 0x78, - 0x74, 0x15, 0xff, 0x1d, 0xc6, 0x47, 0xab, 0xcc, 0xed, 0xd2, 0x88, 0x59, 0xdd, 0x9e, 0x24, 0x78, - 0xd0, 0x71, 0xd9, 0x71, 0x7c, 0xb8, 0x62, 0x07, 0xdd, 0xd5, 0x4e, 0x68, 0x9d, 0xba, 0xcc, 0x62, - 0x6e, 0xe0, 0x5b, 0xde, 0x2a, 0xa3, 0x1e, 0xed, 0x05, 0x21, 0x5b, 0xf5, 0xdc, 0xc3, 0xd5, 0x17, - 0xa1, 0xd5, 0xeb, 0xd1, 0x30, 0xca, 0x3e, 0x04, 0xbb, 0xfe, 0xcb, 0x12, 0xd4, 0xb7, 0x29, 0xed, - 0xad, 0x7b, 0xee, 0x29, 0x25, 0xab, 0x00, 0x7b, 0x34, 0x3c, 0xa5, 0xe1, 0x53, 0xab, 0x4b, 0x9b, + 0x74, 0x15, 0xff, 0x1d, 0xc5, 0xc7, 0xab, 0xcc, 0xed, 0xd2, 0x88, 0x59, 0xdd, 0x9e, 0x24, 0x78, + 0xd0, 0x71, 0xd9, 0x49, 0x7c, 0xb4, 0x62, 0x07, 0xdd, 0xd5, 0x4e, 0x68, 0x9d, 0xb9, 0xcc, 0x62, + 0x6e, 0xe0, 0x5b, 0xde, 0x2a, 0xa3, 0x1e, 0xed, 0x05, 0x21, 0x5b, 0xf5, 0xdc, 0xa3, 0xd5, 0xe7, + 0xa1, 0xd5, 0xeb, 0xd1, 0x30, 0xca, 0x3e, 0x04, 0xbb, 0xfe, 0xcb, 0x12, 0xd4, 0x77, 0x28, 0xed, + 0xad, 0x7b, 0xee, 0x19, 0x25, 0xab, 0x00, 0xfb, 0x34, 0x3c, 0xa3, 0xe1, 0x13, 0xab, 0x4b, 0x9b, 0xa5, 0xe5, 0xd2, 0xdd, 0x7a, 0xeb, 0xf2, 0x20, 0xd1, 0x1a, 0x11, 0x42, 0x4d, 0xdf, 0xea, 0x52, 0x43, 0x21, 0x21, 0xbf, 0x0d, 0x75, 0xfe, 0x1b, 0xf5, 0x2c, 0x9b, 0x36, 0xcb, 0x48, 0x3f, 0x37, - 0x48, 0xb4, 0xba, 0x9f, 0x02, 0x8d, 0x1c, 0x4f, 0xde, 0x81, 0x99, 0x1d, 0x6a, 0x45, 0x74, 0xab, + 0x48, 0xb4, 0xba, 0x9f, 0x02, 0x8d, 0x1c, 0x4f, 0xde, 0x81, 0x99, 0x5d, 0x6a, 0x45, 0x74, 0xbb, 0xdd, 0xac, 0x2c, 0x97, 0xee, 0x56, 0x5a, 0xb3, 0x83, 0x44, 0xab, 0x79, 0x1c, 0x64, 0xba, 0x8e, - 0x91, 0x22, 0xc9, 0x16, 0xcc, 0x6c, 0xbe, 0xec, 0xb9, 0x21, 0x8d, 0x9a, 0xd5, 0xe5, 0xd2, 0xdd, - 0xc6, 0xda, 0xcd, 0x15, 0x61, 0x85, 0x95, 0xd4, 0x0a, 0x2b, 0xfb, 0xa9, 0x15, 0x5a, 0x8b, 0xaf, - 0x12, 0xed, 0xd2, 0x20, 0xd1, 0x66, 0xa8, 0x60, 0xf9, 0xe9, 0xaf, 0xb4, 0x92, 0x91, 0xf2, 0xeb, - 0x7f, 0x53, 0x81, 0xda, 0x13, 0xca, 0x2c, 0xc7, 0x62, 0x16, 0xb9, 0x0d, 0x55, 0x45, 0xaf, 0xda, - 0x20, 0xd1, 0xaa, 0xa8, 0x10, 0x42, 0xc9, 0x5b, 0xa3, 0xaa, 0x4c, 0x0d, 0x12, 0xad, 0xf4, 0x2d, - 0x55, 0x85, 0xdf, 0x87, 0x46, 0x9b, 0x46, 0x76, 0xe8, 0xf6, 0xb8, 0x91, 0x51, 0x8d, 0x7a, 0xeb, - 0xc6, 0x20, 0xd1, 0xae, 0x3a, 0x39, 0xf8, 0x5e, 0xd0, 0x75, 0x19, 0xed, 0xf6, 0x58, 0xdf, 0x50, - 0xa9, 0xc9, 0x0e, 0x4c, 0xef, 0x58, 0x87, 0xd4, 0x8b, 0x9a, 0x53, 0xcb, 0x95, 0xbb, 0x8d, 0xb5, - 0x3b, 0x2b, 0xe9, 0xe2, 0xaf, 0xa4, 0x73, 0x5c, 0x11, 0x04, 0x9b, 0x3e, 0x0b, 0xfb, 0xad, 0xa5, - 0x41, 0xa2, 0x2d, 0x78, 0x08, 0x50, 0x44, 0x4a, 0x19, 0x64, 0x2f, 0xb7, 0xd2, 0xf4, 0xb9, 0x56, - 0x7a, 0xe3, 0x55, 0xa2, 0x95, 0x06, 0x89, 0x76, 0x45, 0x5a, 0x29, 0x97, 0x57, 0xb0, 0x17, 0x59, - 0x86, 0xf2, 0x56, 0xbb, 0x39, 0x83, 0xab, 0xb3, 0x30, 0x48, 0xb4, 0x59, 0xd7, 0x51, 0x86, 0x2e, - 0x6f, 0xb5, 0x6f, 0x7e, 0x00, 0x0d, 0x65, 0x8e, 0x64, 0x01, 0x2a, 0x27, 0xb4, 0x2f, 0x4c, 0x6a, - 0xf0, 0x4f, 0xb2, 0x04, 0x53, 0xa7, 0x96, 0x17, 0x4b, 0x1b, 0x1a, 0xe2, 0xcf, 0x87, 0xe5, 0xef, - 0x94, 0xf4, 0x9f, 0x55, 0xa1, 0x66, 0x04, 0xc2, 0x3f, 0xc9, 0xbb, 0x30, 0xb5, 0xc7, 0x2c, 0x96, - 0xae, 0xc6, 0xe2, 0x20, 0xd1, 0x2e, 0x47, 0x1c, 0xa0, 0x8c, 0x27, 0x28, 0x38, 0xe9, 0xee, 0xb1, - 0x15, 0xa5, 0xab, 0x82, 0xa4, 0x3d, 0x0e, 0x50, 0x49, 0x91, 0x82, 0xbc, 0x03, 0xd5, 0x27, 0x81, - 0x43, 0xe5, 0xc2, 0x90, 0x41, 0xa2, 0xcd, 0x77, 0x03, 0x47, 0x25, 0x44, 0x3c, 0xb9, 0x07, 0xf5, - 0x8d, 0x38, 0x0c, 0xa9, 0xcf, 0xb6, 0xda, 0xe8, 0x64, 0xf5, 0xd6, 0xfc, 0x20, 0xd1, 0xc0, 0x16, - 0x40, 0xee, 0x8e, 0x39, 0x01, 0x37, 0xf5, 0x1e, 0xb3, 0x42, 0x46, 0x9d, 0xe6, 0xd4, 0x44, 0xa6, - 0xe6, 0x0e, 0x79, 0x25, 0x12, 0x2c, 0xc3, 0xa6, 0x96, 0x92, 0xc8, 0x63, 0x68, 0x3c, 0x0a, 0x2d, - 0x9b, 0xee, 0xd2, 0xd0, 0x0d, 0x1c, 0x5c, 0xc3, 0x4a, 0xeb, 0x9d, 0x41, 0xa2, 0x5d, 0xeb, 0x70, - 0xb0, 0xd9, 0x43, 0x78, 0xce, 0xfd, 0x79, 0xa2, 0xd5, 0xda, 0x71, 0x88, 0xd6, 0x33, 0x54, 0x56, - 0xf2, 0xa7, 0x7c, 0x49, 0x22, 0x86, 0xa6, 0xa5, 0x0e, 0xae, 0xde, 0xeb, 0xa7, 0xa8, 0xcb, 0x29, - 0x5e, 0xf3, 0xac, 0x88, 0x99, 0xa1, 0xe0, 0x1b, 0x9a, 0xa7, 0x2a, 0x92, 0x18, 0x50, 0xdb, 0xb3, - 0x8f, 0xa9, 0x13, 0x7b, 0xb4, 0x59, 0x93, 0xe2, 0x33, 0xdf, 0x4d, 0x97, 0x34, 0xa5, 0x68, 0xdd, - 0x94, 0xe2, 0x49, 0x24, 0x21, 0x8a, 0xf9, 0x33, 0x39, 0x1f, 0xd6, 0x3e, 0xfd, 0x27, 0xed, 0xd2, - 0x0f, 0x7f, 0xb9, 0x7c, 0x49, 0xff, 0xb7, 0x32, 0x2c, 0x0c, 0x0b, 0x21, 0x47, 0x30, 0xf7, 0xbc, - 0xe7, 0x58, 0x8c, 0x6e, 0x78, 0x2e, 0xf5, 0x59, 0x84, 0x7e, 0xf2, 0x7a, 0xb5, 0x7e, 0x4b, 0x8e, - 0xdb, 0x8c, 0x91, 0xd1, 0xb4, 0x05, 0xe7, 0x90, 0x62, 0x45, 0xb1, 0xf9, 0x38, 0x22, 0xaa, 0x45, - 0xe8, 0x64, 0x5f, 0x6c, 0x1c, 0x11, 0x1c, 0xcf, 0x18, 0x47, 0x8a, 0x95, 0x3e, 0xe4, 0x3b, 0x87, - 0x7d, 0x74, 0xce, 0xc9, 0x7d, 0x88, 0xb3, 0x8c, 0xf1, 0x21, 0x0e, 0xd6, 0x7f, 0x5d, 0x82, 0x79, - 0x83, 0x46, 0x41, 0x1c, 0xda, 0xf4, 0x31, 0xb5, 0x1c, 0x1a, 0xf2, 0x1d, 0xb0, 0xed, 0xfa, 0x8e, - 0xdc, 0x56, 0xb8, 0x03, 0x4e, 0x5c, 0x5f, 0xdd, 0xc5, 0x88, 0x27, 0xbf, 0x03, 0x33, 0x7b, 0xf1, - 0x21, 0x92, 0x8a, 0x6d, 0x75, 0x0d, 0x57, 0x2c, 0x3e, 0x34, 0x87, 0xc8, 0x53, 0x32, 0xb2, 0x0a, - 0x33, 0x07, 0x34, 0x8c, 0xf2, 0xb8, 0x77, 0x95, 0xcf, 0xf0, 0x54, 0x80, 0x54, 0x06, 0x49, 0x45, - 0xbe, 0x97, 0xc7, 0x5e, 0x19, 0xc8, 0xc9, 0x68, 0xc4, 0xcb, 0xbd, 0xa5, 0x2b, 0x21, 0xaa, 0xb7, - 0xa4, 0x54, 0xfa, 0x5f, 0x95, 0xa1, 0x26, 0x4c, 0x79, 0xb0, 0xc6, 0x03, 0xb9, 0xa2, 0x23, 0x06, - 0x72, 0x3e, 0xe9, 0x2f, 0xad, 0xd9, 0xdb, 0xc3, 0x9a, 0x35, 0x78, 0x42, 0x91, 0x9a, 0xe5, 0xfa, - 0x7c, 0x3c, 0x91, 0x3e, 0x0b, 0x52, 0x9f, 0x5a, 0xaa, 0x4f, 0xae, 0x05, 0xf9, 0x0e, 0x54, 0xf7, - 0x7a, 0xd4, 0x96, 0x51, 0xe4, 0x5a, 0xce, 0x2d, 0x54, 0xe3, 0xb8, 0x83, 0xb5, 0xd6, 0xac, 0x94, - 0x50, 0x8d, 0x7a, 0xd4, 0x36, 0x90, 0x43, 0xd9, 0x2d, 0x3f, 0xaf, 0xc0, 0xac, 0x4a, 0xce, 0xad, - 0xb1, 0xee, 0x38, 0xa1, 0x6a, 0x0d, 0xcb, 0x71, 0x42, 0x03, 0xa1, 0xe4, 0x03, 0x80, 0xdd, 0xf8, - 0xd0, 0x73, 0x6d, 0xa4, 0x29, 0xe7, 0x09, 0xab, 0x87, 0x50, 0x93, 0x93, 0x2a, 0x36, 0x51, 0x88, - 0xc9, 0x5d, 0xa8, 0x3d, 0x0e, 0x22, 0xc6, 0x73, 0xa4, 0xb4, 0x0b, 0x26, 0xec, 0x63, 0x09, 0x33, - 0x32, 0x2c, 0xb1, 0xa0, 0xbe, 0xd1, 0x75, 0x64, 0x72, 0xab, 0x62, 0x72, 0x7b, 0x7b, 0xbc, 0x72, - 0x2b, 0x19, 0x9d, 0xc8, 0x71, 0xb7, 0xa5, 0xae, 0x4b, 0x76, 0xd7, 0x31, 0x47, 0x72, 0x5d, 0x2e, - 0x95, 0x3b, 0x53, 0x1a, 0x23, 0xa4, 0xf9, 0xc8, 0x68, 0x08, 0xca, 0x9d, 0x29, 0x94, 0x10, 0xd5, - 0x99, 0xb2, 0xdc, 0xf3, 0x3e, 0xd4, 0x9f, 0x47, 0x74, 0x3f, 0xf6, 0x7d, 0xea, 0x61, 0xe0, 0xad, - 0xb5, 0x9a, 0x7c, 0x0e, 0x71, 0x44, 0x4d, 0x86, 0x50, 0x75, 0x0e, 0x19, 0xe9, 0xcd, 0x03, 0x98, - 0x2f, 0x4e, 0x7f, 0x4c, 0xfa, 0x5b, 0x51, 0xd3, 0x5f, 0x63, 0xad, 0x99, 0x4f, 0x72, 0x23, 0xe8, - 0x76, 0x2d, 0x5f, 0xb0, 0x1f, 0xac, 0xa9, 0x89, 0xf1, 0x47, 0x25, 0x98, 0x2f, 0x62, 0xc9, 0x0a, - 0x4c, 0xcb, 0xc4, 0x50, 0xc2, 0xc4, 0xc0, 0x7d, 0x78, 0x5a, 0xa4, 0x84, 0x42, 0x22, 0x90, 0x54, - 0xdc, 0x85, 0xa5, 0x84, 0x66, 0x79, 0xb9, 0x92, 0xba, 0xb0, 0x2d, 0x40, 0x46, 0x8a, 0x23, 0x3a, - 0x4c, 0x1b, 0x34, 0x8a, 0x3d, 0x26, 0x17, 0x14, 0xb8, 0xd8, 0x10, 0x21, 0x86, 0xc4, 0xe8, 0x3f, - 0x00, 0xd8, 0xdf, 0xd9, 0xdb, 0xa6, 0xfd, 0x5d, 0xcb, 0xc5, 0x78, 0xb2, 0x41, 0x43, 0x86, 0xd3, - 0x98, 0x15, 0xf1, 0xc4, 0xa6, 0x21, 0x53, 0xe3, 0x09, 0xc7, 0x93, 0xb7, 0xa0, 0xb2, 0x4d, 0xfb, - 0xa8, 0xf5, 0x6c, 0xeb, 0xca, 0x20, 0xd1, 0xe6, 0x4e, 0xa8, 0x12, 0xb7, 0x0c, 0x8e, 0xd5, 0x7f, - 0x56, 0x86, 0xcb, 0x9c, 0x7a, 0x3d, 0x66, 0xc7, 0x41, 0xe8, 0xb2, 0xfe, 0x45, 0xde, 0xcc, 0x1f, - 0x15, 0x36, 0xf3, 0x1b, 0xca, 0x42, 0xab, 0x1a, 0x4e, 0xb4, 0xa7, 0xff, 0xba, 0x0a, 0x8b, 0x63, - 0xb8, 0xc8, 0x3d, 0xa8, 0xee, 0xf7, 0x7b, 0x69, 0x8d, 0xc4, 0x7d, 0xb4, 0xca, 0x0f, 0x0f, 0x9f, - 0x27, 0xda, 0x6c, 0x4a, 0xce, 0xf1, 0x06, 0x52, 0x91, 0x35, 0x68, 0x6c, 0x78, 0x71, 0xc4, 0x64, - 0xf9, 0x2e, 0xec, 0x85, 0x55, 0x9c, 0x2d, 0xc0, 0xa2, 0x7e, 0x57, 0x89, 0xc8, 0x7b, 0x30, 0xbb, - 0x71, 0x4c, 0xed, 0x13, 0xd7, 0xef, 0x6c, 0xd3, 0x7e, 0xd4, 0xac, 0x2c, 0x57, 0xd2, 0xf5, 0xb3, - 0x25, 0xdc, 0x3c, 0xa1, 0xfd, 0xc8, 0x28, 0x90, 0x91, 0xef, 0x42, 0x63, 0xcf, 0xed, 0xf8, 0x29, - 0x57, 0x15, 0xb9, 0x6e, 0xf2, 0x92, 0x22, 0x12, 0x60, 0x64, 0x52, 0x0b, 0x61, 0x85, 0x9c, 0x17, - 0x74, 0x46, 0xe0, 0x51, 0x51, 0x07, 0xcb, 0x82, 0x2e, 0xe4, 0x00, 0xb5, 0xa0, 0x43, 0x0a, 0xb2, - 0x0d, 0x33, 0xfc, 0xe3, 0x89, 0xd5, 0x6b, 0x4e, 0x63, 0x5c, 0xb9, 0xaa, 0xee, 0x7a, 0x44, 0xf4, - 0x5c, 0xbf, 0xa3, 0x6e, 0x7c, 0x8f, 0x9a, 0x5d, 0xab, 0xa7, 0xba, 0x86, 0x24, 0x24, 0xdf, 0x87, - 0x46, 0xee, 0xd9, 0x51, 0x73, 0x06, 0x05, 0x2e, 0xe5, 0x02, 0x73, 0x64, 0x4b, 0x93, 0xf2, 0xae, - 0x33, 0x2f, 0xe2, 0xba, 0x98, 0x3d, 0xce, 0xa2, 0x2a, 0xa4, 0x48, 0x2a, 0x04, 0xa7, 0xda, 0x6b, - 0x83, 0x53, 0xe9, 0xbc, 0xe0, 0xa4, 0x1b, 0xd0, 0x50, 0x14, 0x13, 0x3b, 0xb6, 0x1b, 0x64, 0x85, - 0xb2, 0xdc, 0xb1, 0x1c, 0x62, 0x48, 0x0c, 0xd1, 0x60, 0x6a, 0x27, 0xb0, 0x2d, 0x4f, 0x6e, 0xfd, - 0xfa, 0x20, 0xd1, 0xa6, 0x3c, 0x0e, 0x30, 0x04, 0x5c, 0xff, 0xaf, 0x12, 0x2c, 0xec, 0x86, 0xc1, - 0xa9, 0xcb, 0x5d, 0x7f, 0x3f, 0x38, 0xa1, 0xfe, 0xc1, 0xb7, 0xc9, 0x56, 0xba, 0x0a, 0x25, 0xe4, - 0xba, 0xcf, 0xb9, 0x70, 0x15, 0x3e, 0x4f, 0xb4, 0x77, 0xce, 0x3d, 0x55, 0xa2, 0xf5, 0xd3, 0x55, - 0x52, 0xce, 0x22, 0xe5, 0xc9, 0x8b, 0x9b, 0x73, 0xce, 0x22, 0x1a, 0x4c, 0xe1, 0x54, 0xe5, 0x36, - 0x46, 0xad, 0x18, 0x07, 0x18, 0x02, 0xae, 0xec, 0x9f, 0xbf, 0x2f, 0x8f, 0xe8, 0x77, 0x81, 0x03, - 0xcb, 0xc7, 0x85, 0xc0, 0xa2, 0x9c, 0x12, 0x8b, 0x2a, 0x4e, 0x14, 0x59, 0x2c, 0x58, 0x1a, 0xc7, - 0xf5, 0x15, 0x2e, 0xbe, 0xfe, 0x77, 0x65, 0x98, 0xe7, 0x07, 0x35, 0xd7, 0xc6, 0x01, 0xa2, 0x8b, - 0x6c, 0xfa, 0x3f, 0x28, 0x98, 0xfe, 0xb6, 0x52, 0xc3, 0x28, 0x0a, 0x4e, 0x64, 0xf8, 0x13, 0x20, - 0xa3, 0x3c, 0xe4, 0x39, 0xcc, 0xaa, 0x50, 0xb4, 0x7e, 0xe1, 0x30, 0x35, 0xbc, 0x4b, 0x5b, 0x57, - 0xe5, 0x28, 0x73, 0x11, 0xf2, 0x99, 0xb8, 0x03, 0x22, 0xa3, 0x20, 0x46, 0xff, 0xdb, 0x32, 0xcc, - 0x29, 0x51, 0xfd, 0x22, 0xaf, 0xc0, 0x83, 0xc2, 0x0a, 0xdc, 0x52, 0xb2, 0x6a, 0xae, 0xdf, 0x44, - 0x0b, 0xf0, 0x08, 0xae, 0x8c, 0xb0, 0x0c, 0xa7, 0xc8, 0xd2, 0x04, 0x29, 0x52, 0x14, 0x2d, 0xe2, - 0xff, 0x46, 0xe0, 0x1f, 0xb9, 0x9d, 0x83, 0xfb, 0x5f, 0xc7, 0xa2, 0x45, 0xd5, 0x10, 0xad, 0x75, - 0xff, 0x1c, 0x03, 0xff, 0x64, 0x0a, 0x16, 0xc7, 0x70, 0x91, 0x75, 0x58, 0xd8, 0xa3, 0x11, 0x4e, - 0x9c, 0xda, 0x41, 0xe8, 0xb8, 0x7e, 0x47, 0xda, 0x09, 0x0f, 0x8c, 0x91, 0xc0, 0x99, 0x61, 0x8a, - 0x34, 0x46, 0xc8, 0xb1, 0x3d, 0x23, 0x24, 0x6f, 0xb5, 0xa5, 0x09, 0x45, 0x7b, 0x46, 0x2e, 0x12, - 0xb6, 0x67, 0x52, 0x02, 0xb2, 0x03, 0x8b, 0xbb, 0x61, 0xf0, 0xb2, 0x8f, 0x15, 0x4a, 0xc4, 0x0f, - 0x25, 0xb2, 0x94, 0xe1, 0x7c, 0x58, 0x94, 0xf4, 0x38, 0xda, 0xc4, 0x82, 0x26, 0x32, 0xf9, 0xf9, - 0x45, 0xd4, 0x34, 0xe3, 0xd8, 0xc8, 0x87, 0x30, 0xb5, 0x1e, 0x3b, 0x2e, 0x93, 0x06, 0x56, 0xea, - 0x0d, 0x04, 0x0b, 0x55, 0x5b, 0x73, 0xd2, 0x34, 0x53, 0x16, 0x07, 0x1a, 0x82, 0x85, 0x7c, 0xc2, - 0x7d, 0xce, 0xa5, 0x3e, 0xdb, 0x72, 0x3c, 0xca, 0x33, 0x5e, 0x10, 0x33, 0x34, 0x75, 0xa5, 0xf5, - 0xd6, 0x20, 0xd1, 0x16, 0x45, 0x47, 0xc2, 0x74, 0x1d, 0x8f, 0x9a, 0x4c, 0xa0, 0x0b, 0xd5, 0xfc, - 0x28, 0x37, 0xf9, 0x01, 0x5c, 0x6d, 0xbb, 0x91, 0x1d, 0xf8, 0x3e, 0xb5, 0x99, 0x48, 0x8d, 0x0e, - 0x16, 0xe4, 0xe2, 0xdc, 0xc2, 0xc5, 0x5e, 0x77, 0x32, 0x02, 0x53, 0xe4, 0x54, 0xc7, 0xe4, 0x35, - 0xfa, 0xe7, 0x89, 0x56, 0x6d, 0x05, 0x81, 0x67, 0x8c, 0x97, 0xc0, 0x67, 0x9b, 0xb5, 0x7e, 0xb7, - 0x7c, 0x46, 0xc3, 0x53, 0xcb, 0x93, 0xbd, 0x3f, 0x9c, 0xed, 0x09, 0xa5, 0x3d, 0xd3, 0xe2, 0x58, - 0xd3, 0x95, 0xe8, 0xe2, 0x6c, 0x47, 0xb8, 0xc9, 0x43, 0x45, 0xe4, 0x46, 0x10, 0xfb, 0xec, 0x89, - 0xf5, 0x12, 0x2b, 0xa2, 0x8a, 0x38, 0x61, 0x29, 0x22, 0x6d, 0x8e, 0x36, 0xbb, 0xd6, 0x4b, 0x63, - 0x94, 0x85, 0xfc, 0x2e, 0xd4, 0xb1, 0x72, 0xe1, 0x15, 0x6e, 0xb3, 0x8e, 0x9a, 0xf2, 0x3d, 0x04, - 0x58, 0xd5, 0x98, 0x56, 0xcc, 0x8e, 0x33, 0xe5, 0x72, 0x42, 0xfd, 0xd3, 0x0a, 0x34, 0x94, 0x45, - 0xe2, 0x67, 0x17, 0xa5, 0x7c, 0xc6, 0xb3, 0x0b, 0x2f, 0x9f, 0xd5, 0xb3, 0x0b, 0x16, 0xce, 0xf7, - 0x78, 0x8d, 0xd5, 0xe1, 0x9b, 0x4f, 0xf8, 0x1a, 0x36, 0x5e, 0x43, 0x84, 0xa8, 0x8d, 0x57, 0x41, - 0x43, 0x76, 0x60, 0x01, 0x07, 0x91, 0x5e, 0x1b, 0x3d, 0x37, 0xb6, 0xa4, 0xaf, 0x2d, 0x0f, 0x12, - 0xed, 0x36, 0x3a, 0x84, 0x29, 0xbd, 0x3c, 0x32, 0xe3, 0xd0, 0x55, 0x64, 0x8c, 0x70, 0x92, 0x7f, - 0x2c, 0xc1, 0x3c, 0x02, 0x37, 0x4f, 0xa9, 0xcf, 0x50, 0x58, 0x55, 0x76, 0x07, 0xb2, 0x56, 0xfd, - 0x1e, 0x0b, 0x5d, 0xbf, 0x73, 0xc0, 0xcf, 0x8b, 0x51, 0xeb, 0x8f, 0xb8, 0xe7, 0xfd, 0x22, 0xd1, - 0xde, 0xff, 0x62, 0x8d, 0x7f, 0x29, 0x24, 0x1a, 0x24, 0xda, 0x4d, 0x31, 0x45, 0x8a, 0x03, 0x0e, - 0x4d, 0x70, 0x68, 0x2e, 0xe4, 0xa1, 0x9c, 0xdd, 0xbe, 0x75, 0xe8, 0x51, 0x8c, 0x99, 0x53, 0xa8, - 0xea, 0x9d, 0x5c, 0x0e, 0xe3, 0x28, 0x8c, 0x9b, 0x23, 0x72, 0x32, 0x2e, 0xfd, 0xff, 0x4a, 0x4a, - 0x7b, 0xfd, 0xe2, 0x86, 0xcf, 0x0f, 0x0a, 0xe1, 0xf3, 0x7a, 0xce, 0x9d, 0xe9, 0xc6, 0xd1, 0xe3, - 0x02, 0xa7, 0x7e, 0x19, 0xe6, 0x0a, 0x44, 0x98, 0x57, 0xd6, 0x6d, 0x9b, 0x46, 0x91, 0x41, 0xff, - 0x2c, 0xa6, 0x11, 0xfb, 0x5a, 0xe6, 0x95, 0x82, 0x86, 0x13, 0xe5, 0x95, 0xff, 0x28, 0xc3, 0xe2, - 0x18, 0x2e, 0x6e, 0x9b, 0xe7, 0x11, 0x2d, 0xf4, 0xb9, 0xe2, 0x88, 0x86, 0x06, 0x42, 0xf9, 0x69, - 0x41, 0x14, 0xb4, 0xca, 0x19, 0x08, 0x0b, 0xda, 0xf4, 0x8c, 0xb2, 0x9e, 0x5e, 0x38, 0x70, 0x43, - 0xcc, 0xab, 0xcd, 0xb7, 0x74, 0x18, 0x8e, 0x7d, 0xed, 0x45, 0xc4, 0x1e, 0xcc, 0x6c, 0x84, 0x14, - 0x9b, 0xec, 0xd5, 0xc9, 0x8f, 0x39, 0xb6, 0x60, 0x19, 0x3e, 0xe6, 0x48, 0x49, 0xea, 0xd9, 0x69, - 0xea, 0xab, 0x3a, 0x3b, 0xe9, 0x7f, 0x59, 0x1a, 0xb2, 0xe1, 0x43, 0xd7, 0x63, 0x34, 0x24, 0xd7, - 0xf0, 0x7e, 0x47, 0x58, 0x70, 0x7a, 0x90, 0x68, 0x65, 0xd7, 0x31, 0xca, 0x5b, 0xed, 0xcc, 0xb6, - 0xe5, 0xb1, 0xb6, 0xfd, 0xbd, 0xc9, 0x4c, 0x87, 0x36, 0x47, 0xd3, 0x49, 0x83, 0xe9, 0x7f, 0x51, - 0x86, 0x69, 0x6e, 0xfd, 0x8b, 0xec, 0xd9, 0xef, 0x17, 0x3c, 0x7b, 0xa9, 0xd8, 0x7e, 0x98, 0xc8, - 0xa1, 0x7f, 0x5d, 0x02, 0xc8, 0x89, 0xc9, 0xf7, 0x60, 0xe6, 0x19, 0x5e, 0x08, 0xa6, 0x77, 0x1a, - 0x43, 0x2d, 0x0d, 0x89, 0x6c, 0xdd, 0x48, 0xd7, 0x3a, 0x10, 0x00, 0xd5, 0x0a, 0x92, 0x86, 0x3c, - 0x82, 0xa9, 0x75, 0xcf, 0x0b, 0x5e, 0x8c, 0x76, 0x1b, 0xb9, 0xa4, 0x8d, 0xc0, 0x77, 0x5c, 0x21, - 0xec, 0xba, 0x14, 0x76, 0xd9, 0xe2, 0xe4, 0xaa, 0x6b, 0x23, 0x3f, 0x69, 0x43, 0xb5, 0x4d, 0xfd, - 0xf4, 0x6e, 0xe2, 0x6c, 0x39, 0xd7, 0xa4, 0x9c, 0x79, 0x87, 0xfa, 0x6a, 0x7b, 0x0f, 0xb9, 0xf5, - 0x9f, 0x56, 0x45, 0xf3, 0x22, 0x9d, 0xde, 0x03, 0x98, 0x7d, 0x18, 0x84, 0x2f, 0xac, 0xd0, 0x59, - 0xef, 0x50, 0x5f, 0x34, 0x11, 0x6b, 0xd8, 0x7e, 0x9e, 0x3b, 0x12, 0x70, 0xd3, 0xe2, 0x88, 0x2c, - 0x99, 0x17, 0xc8, 0xc9, 0x33, 0x98, 0x7b, 0x62, 0xbd, 0x94, 0xd9, 0x72, 0x7f, 0x7f, 0x07, 0xb5, - 0xac, 0xb4, 0xde, 0x1d, 0x24, 0xda, 0x8d, 0xae, 0xf5, 0x32, 0x4d, 0xb2, 0x26, 0x63, 0xde, 0x19, - 0xf7, 0x64, 0x45, 0x7e, 0xe2, 0xc1, 0xfc, 0x6e, 0x10, 0x32, 0x39, 0x08, 0x2f, 0x4c, 0x85, 0xbe, - 0x8b, 0xb9, 0xbe, 0x7c, 0x1a, 0x98, 0x69, 0x5b, 0xab, 0xaf, 0x12, 0xad, 0xf4, 0x8b, 0x44, 0x03, - 0x0e, 0x12, 0x1a, 0xf1, 0x81, 0x79, 0x66, 0x35, 0x8f, 0x32, 0x09, 0x6a, 0xce, 0x2b, 0xca, 0x26, - 0x0f, 0xe0, 0x0a, 0xaf, 0xb3, 0xdc, 0x23, 0xd7, 0xb6, 0x18, 0x7d, 0x18, 0x84, 0x5d, 0x8b, 0xc9, - 0xcb, 0x46, 0xbc, 0x54, 0xe7, 0x35, 0x1a, 0x97, 0xd4, 0xb5, 0x98, 0x31, 0x4a, 0x49, 0xfe, 0xf0, - 0xec, 0x62, 0xf2, 0x5b, 0x83, 0x44, 0x7b, 0x63, 0x4c, 0x31, 0x79, 0x86, 0x15, 0xc6, 0x94, 0x95, - 0x9d, 0xd7, 0x97, 0x95, 0xdf, 0x96, 0xad, 0xaa, 0x37, 0xcf, 0x28, 0x2d, 0x0b, 0x03, 0xbd, 0xae, - 0xc8, 0xd4, 0x7f, 0x52, 0x81, 0xf9, 0xa2, 0x0f, 0x11, 0x1d, 0xa6, 0x77, 0x82, 0x8e, 0xeb, 0xa7, - 0xcd, 0x07, 0x6c, 0x69, 0x79, 0x08, 0x31, 0x24, 0x86, 0xbc, 0x0d, 0x90, 0x65, 0xcb, 0x34, 0xa6, - 0xcb, 0xeb, 0x78, 0x05, 0x41, 0xfe, 0x04, 0xe0, 0x69, 0xe0, 0x50, 0x79, 0xf3, 0x50, 0x91, 0xdb, - 0x29, 0xab, 0x79, 0x44, 0x37, 0x5d, 0xd4, 0x4d, 0xdf, 0x90, 0x75, 0x93, 0xbc, 0x3f, 0x1f, 0x24, - 0xda, 0x55, 0x3f, 0x70, 0xe8, 0xe8, 0xa5, 0x83, 0x22, 0x91, 0x3c, 0x80, 0x29, 0x23, 0xf6, 0x68, - 0x7a, 0xa9, 0x31, 0xaf, 0xec, 0x8b, 0xd8, 0xa3, 0xf9, 0xae, 0x0a, 0xe3, 0xe1, 0xee, 0x25, 0x07, - 0x90, 0x8f, 0x00, 0xb6, 0xe3, 0x43, 0xfa, 0x28, 0x0c, 0xe2, 0x5e, 0xda, 0xed, 0xd4, 0x06, 0x89, - 0x76, 0xeb, 0x24, 0x3e, 0xa4, 0xa1, 0x4f, 0x19, 0x8d, 0xcc, 0x0e, 0x22, 0xd5, 0xf1, 0x73, 0x16, - 0x62, 0xc0, 0x8c, 0x0c, 0xb1, 0xf2, 0x92, 0xff, 0xcd, 0x33, 0x32, 0xab, 0xb2, 0x45, 0xf1, 0x94, - 0x15, 0x0a, 0x70, 0xa1, 0x0b, 0x2a, 0x40, 0x7a, 0x1b, 0xae, 0x9f, 0xc1, 0x9a, 0x37, 0x66, 0x4b, - 0xe7, 0x35, 0x66, 0xf5, 0xff, 0x2c, 0x41, 0x95, 0x2b, 0x49, 0xde, 0x83, 0x7a, 0x7a, 0x05, 0x99, - 0xf2, 0x5d, 0xe7, 0xa7, 0x87, 0x30, 0x05, 0xaa, 0x77, 0x29, 0x19, 0x25, 0x1f, 0xea, 0x80, 0x86, - 0x87, 0xe9, 0xda, 0xe2, 0x50, 0xa7, 0x1c, 0xa0, 0x0e, 0x85, 0x14, 0x9c, 0xf4, 0xfb, 0xc7, 0x34, - 0x4c, 0x2f, 0xa1, 0x90, 0xf4, 0x05, 0x07, 0xa8, 0xa4, 0x48, 0x41, 0x56, 0x61, 0x66, 0xdd, 0x16, - 0xb1, 0xb5, 0x8a, 0x72, 0xd1, 0x18, 0x96, 0x3d, 0x12, 0x40, 0x25, 0x95, 0xfe, 0x26, 0xd4, 0xb3, - 0x1d, 0x4f, 0x96, 0x60, 0x0a, 0x3f, 0x44, 0x9c, 0x32, 0xc4, 0x1f, 0x4c, 0x62, 0x3c, 0x0d, 0x5e, - 0xe4, 0xae, 0xca, 0x99, 0x49, 0x8c, 0x2b, 0x36, 0x51, 0x3b, 0xe5, 0x5f, 0xa7, 0x01, 0x72, 0x62, - 0x42, 0x61, 0xfe, 0xd9, 0x56, 0x7b, 0x63, 0xcb, 0xa1, 0x3e, 0x73, 0x99, 0x4b, 0xc7, 0xb4, 0xb2, - 0x36, 0x5f, 0x32, 0x1a, 0xfa, 0x96, 0x27, 0x69, 0xfa, 0xad, 0x37, 0xe5, 0x00, 0x37, 0x02, 0xd7, - 0xb1, 0x4d, 0x37, 0x63, 0x55, 0x43, 0x68, 0x51, 0x28, 0x1f, 0x66, 0x6f, 0xfd, 0xc9, 0x8e, 0x32, - 0x4c, 0x79, 0xf2, 0x61, 0x22, 0xab, 0xeb, 0x9d, 0x31, 0x4c, 0x51, 0x28, 0x39, 0x81, 0x85, 0x47, - 0x78, 0x9a, 0x52, 0x06, 0xaa, 0x9c, 0x3b, 0xd0, 0x5b, 0x72, 0xa0, 0x5b, 0xe2, 0x24, 0x36, 0x7e, - 0xa8, 0x11, 0xc1, 0xf9, 0x26, 0xab, 0x9e, 0x7b, 0xfb, 0xf1, 0xc3, 0x12, 0x4c, 0xef, 0x87, 0x96, - 0xcb, 0xd2, 0xda, 0xf0, 0x8c, 0xd8, 0xf6, 0x89, 0x8c, 0x6d, 0xef, 0x7d, 0xc1, 0x33, 0xa1, 0x90, - 0xcd, 0x4f, 0xbb, 0x0c, 0xbf, 0xd4, 0xd3, 0xae, 0xc0, 0x91, 0x47, 0x30, 0xcd, 0x6b, 0xb9, 0x38, - 0x7d, 0x65, 0xa4, 0x14, 0x2b, 0x18, 0xaa, 0x05, 0xb2, 0xd5, 0x94, 0xb6, 0x58, 0x88, 0xf0, 0xbf, - 0x2a, 0x48, 0x50, 0xa8, 0xaf, 0xba, 0x66, 0x7e, 0xb3, 0x57, 0x5d, 0xe4, 0x19, 0xd4, 0x65, 0xf5, - 0xdc, 0xea, 0xcb, 0xfb, 0x16, 0x25, 0x83, 0x67, 0x28, 0xe5, 0x72, 0x59, 0x80, 0x4c, 0xf5, 0x25, - 0x85, 0x91, 0xcb, 0x20, 0xc6, 0x70, 0xbb, 0xa1, 0xb0, 0xf0, 0x19, 0x6a, 0x8f, 0xda, 0x21, 0x65, - 0x91, 0x68, 0x65, 0xe4, 0xad, 0x08, 0x55, 0x66, 0xde, 0x8c, 0xf8, 0x71, 0x09, 0x16, 0x86, 0x5d, - 0x86, 0x7c, 0x17, 0x1a, 0x1b, 0x22, 0x47, 0x06, 0x61, 0x56, 0x88, 0x63, 0x8b, 0xca, 0x4e, 0xc1, - 0x66, 0xe1, 0xc9, 0x95, 0x4a, 0x4e, 0xd6, 0xa0, 0xc6, 0xb7, 0xa0, 0x9f, 0xdf, 0xee, 0x61, 0x84, - 0x89, 0x25, 0x4c, 0xbd, 0x4e, 0x4a, 0xe9, 0x94, 0x1d, 0xfc, 0xef, 0x65, 0x68, 0x28, 0x4b, 0x46, - 0xde, 0x85, 0xda, 0x56, 0xb4, 0x13, 0xd8, 0x27, 0xd4, 0x91, 0x85, 0x19, 0x3e, 0xdd, 0x73, 0x23, - 0xd3, 0x43, 0xa0, 0x91, 0xa1, 0x49, 0x0b, 0xe6, 0xc4, 0xd7, 0x13, 0x1a, 0x45, 0x56, 0x27, 0x1d, - 0xfd, 0xf6, 0x20, 0xd1, 0x9a, 0x82, 0xd8, 0xec, 0x0a, 0x8c, 0x32, 0x87, 0x22, 0x0b, 0xf9, 0x63, - 0x00, 0x01, 0xe0, 0xab, 0x3c, 0xc1, 0x1b, 0x98, 0x74, 0x1b, 0x5f, 0x95, 0x03, 0xf0, 0x12, 0x67, - 0xe8, 0xb8, 0xa3, 0x08, 0xc4, 0x47, 0x50, 0x81, 0x7d, 0x32, 0xf9, 0xc3, 0xc1, 0xfc, 0x11, 0x54, - 0x60, 0x9f, 0x98, 0xe3, 0xcf, 0x53, 0xaa, 0x48, 0xfd, 0x57, 0x25, 0xc5, 0xed, 0xc8, 0x27, 0x50, - 0xcf, 0x96, 0x46, 0xd6, 0xf1, 0xd7, 0xd4, 0xbb, 0x7e, 0x89, 0x32, 0xe8, 0x51, 0xeb, 0x96, 0x2c, - 0xa6, 0x16, 0xb3, 0x35, 0x2e, 0x78, 0x61, 0x0a, 0x24, 0x1f, 0x43, 0x15, 0x6d, 0x73, 0xfe, 0x15, - 0x5a, 0x1a, 0xea, 0xab, 0xdc, 0x28, 0x38, 0x53, 0xe4, 0x24, 0xf7, 0xe5, 0x31, 0x4e, 0x58, 0xf7, - 0x4a, 0x31, 0xcc, 0xf3, 0xa9, 0x64, 0x31, 0x3e, 0x3f, 0xdd, 0x29, 0x1e, 0xf2, 0xe7, 0x25, 0x58, - 0x7c, 0xbe, 0xf6, 0xd0, 0xa0, 0x1d, 0x37, 0x62, 0xa2, 0x76, 0x6c, 0xf3, 0xec, 0x71, 0x03, 0x2a, - 0x86, 0xf5, 0x42, 0x3e, 0x01, 0x98, 0x19, 0x24, 0x5a, 0x25, 0xb4, 0x5e, 0x18, 0x1c, 0x46, 0xee, - 0x41, 0x7d, 0x9b, 0xf6, 0x1f, 0x5b, 0xbe, 0xe3, 0x51, 0x79, 0xf9, 0x8f, 0x9d, 0xda, 0x13, 0xda, - 0x37, 0x8f, 0x11, 0x6a, 0xe4, 0x04, 0xbc, 0xf2, 0xdb, 0x8d, 0x0f, 0xb7, 0xa9, 0x38, 0x67, 0xcc, - 0x8a, 0xca, 0xaf, 0x17, 0x1f, 0x9e, 0xd0, 0xbe, 0x21, 0x31, 0xfa, 0x3f, 0x97, 0x61, 0x61, 0x78, - 0xc7, 0x91, 0x8f, 0x60, 0x76, 0xd7, 0x8a, 0xa2, 0x17, 0x41, 0xe8, 0x3c, 0xb6, 0xa2, 0x63, 0x39, - 0x95, 0x5b, 0x83, 0x44, 0xbb, 0xde, 0x93, 0x70, 0xf3, 0xd8, 0x8a, 0xd4, 0xad, 0x58, 0x60, 0xe0, - 0xb9, 0x79, 0xff, 0xd9, 0xfe, 0x6e, 0xfa, 0x44, 0x41, 0xee, 0x1c, 0x16, 0xb0, 0x9e, 0x59, 0x7c, - 0xa7, 0x90, 0x92, 0x91, 0x0e, 0x5c, 0x1e, 0xb2, 0x85, 0x34, 0xab, 0xd2, 0xdc, 0x18, 0x63, 0x2c, - 0xd1, 0x19, 0x8b, 0xd7, 0x8e, 0xcc, 0x50, 0xc1, 0x28, 0x03, 0x0c, 0x4b, 0x25, 0x1f, 0x00, 0x3c, - 0x5f, 0x7b, 0x88, 0xad, 0x4f, 0x1a, 0xa2, 0xe3, 0xce, 0x89, 0x17, 0x3a, 0x5c, 0x88, 0x2d, 0xc0, - 0x6a, 0x79, 0x98, 0x13, 0xeb, 0x3e, 0xcc, 0xaa, 0x9e, 0xc6, 0xeb, 0x13, 0xa5, 0xe1, 0x59, 0x4b, - 0xdf, 0x0b, 0xc8, 0x36, 0xa7, 0x38, 0xfc, 0x97, 0x47, 0x0e, 0xff, 0x77, 0xa1, 0x96, 0x06, 0x28, - 0xf5, 0x9d, 0x8f, 0x4c, 0x67, 0x7d, 0x23, 0xc3, 0xea, 0xdf, 0x80, 0x19, 0xe9, 0x49, 0xaf, 0x7f, - 0x4c, 0xab, 0xff, 0xa8, 0x0c, 0x97, 0x0d, 0xca, 0x0b, 0x19, 0xf9, 0x76, 0xe6, 0x6b, 0xf9, 0xd0, - 0xa3, 0xa0, 0xe1, 0xd9, 0x55, 0x94, 0xfe, 0x2f, 0x25, 0x58, 0x1c, 0x43, 0xfb, 0x65, 0x6e, 0xa3, - 0xc8, 0xfb, 0x50, 0x6f, 0xbb, 0x96, 0xb7, 0xee, 0x38, 0x61, 0x5a, 0x3b, 0x63, 0x3a, 0x72, 0x5c, - 0x9e, 0x8d, 0x38, 0x54, 0x0d, 0x2e, 0x19, 0x29, 0xf9, 0xa6, 0x74, 0x8d, 0x4a, 0x66, 0xdc, 0xf4, - 0x29, 0x09, 0x88, 0x39, 0xe5, 0x0f, 0x49, 0xf4, 0x7f, 0x28, 0x03, 0x11, 0x40, 0xe9, 0x5d, 0x6e, - 0x70, 0xa1, 0x2f, 0xd4, 0x5b, 0x85, 0x05, 0x5c, 0x56, 0x1e, 0x7c, 0x0c, 0x29, 0x39, 0x51, 0x25, - 0xfc, 0xe3, 0x32, 0x5c, 0x1b, 0xcf, 0xf8, 0xa5, 0x16, 0xf4, 0x1e, 0xd4, 0xf1, 0x1a, 0x4a, 0x79, - 0xb3, 0x83, 0x11, 0x54, 0xdc, 0x59, 0x21, 0x7d, 0x4e, 0x40, 0x8e, 0x60, 0x6e, 0xc7, 0x8a, 0xd8, - 0x63, 0x6a, 0x85, 0xec, 0x90, 0x5a, 0x6c, 0x82, 0x44, 0x9a, 0x3d, 0x57, 0xc5, 0xd7, 0xbe, 0xc7, - 0x29, 0xe7, 0xf0, 0x73, 0xd5, 0x82, 0xd8, 0xcc, 0x5d, 0xaa, 0xe7, 0xbb, 0xcb, 0x37, 0x3f, 0x82, - 0x59, 0xb5, 0x0f, 0x48, 0x6a, 0x50, 0x7d, 0xfa, 0xec, 0xe9, 0xe6, 0xc2, 0x25, 0xd2, 0x80, 0x99, - 0xdd, 0xcd, 0xa7, 0xed, 0xad, 0xa7, 0x8f, 0x16, 0x4a, 0x64, 0x16, 0x6a, 0xeb, 0xbb, 0xbb, 0xc6, - 0xb3, 0x83, 0xcd, 0xf6, 0x42, 0x99, 0x00, 0x4c, 0xb7, 0x37, 0x9f, 0x6e, 0x6d, 0xb6, 0x17, 0x2a, - 0xad, 0xa5, 0x57, 0xff, 0x73, 0xe7, 0xd2, 0xab, 0xcf, 0xee, 0x94, 0x7e, 0xfe, 0xd9, 0x9d, 0xd2, - 0x7f, 0x7f, 0x76, 0xa7, 0xf4, 0xe9, 0xff, 0xde, 0xb9, 0x74, 0x38, 0x8d, 0xba, 0xdc, 0xff, 0xff, - 0x00, 0x00, 0x00, 0xff, 0xff, 0x00, 0xab, 0x4c, 0x93, 0x30, 0x31, 0x00, 0x00, + 0x91, 0x22, 0xc9, 0x36, 0xcc, 0x6c, 0xbe, 0xe8, 0xb9, 0x21, 0x8d, 0x9a, 0xd5, 0xe5, 0xd2, 0xdd, + 0xc6, 0xda, 0xcd, 0x15, 0x61, 0x85, 0x95, 0xd4, 0x0a, 0x2b, 0x07, 0xa9, 0x15, 0x5a, 0x8b, 0x2f, + 0x13, 0xed, 0xd2, 0x20, 0xd1, 0x66, 0xa8, 0x60, 0xf9, 0xdb, 0x5f, 0x69, 0x25, 0x23, 0xe5, 0xd7, + 0xff, 0xa6, 0x02, 0xb5, 0xc7, 0x94, 0x59, 0x8e, 0xc5, 0x2c, 0x72, 0x1b, 0xaa, 0x8a, 0x5e, 0xb5, + 0x41, 0xa2, 0x55, 0x51, 0x21, 0x84, 0x92, 0xb7, 0x46, 0x55, 0x99, 0x1a, 0x24, 0x5a, 0xe9, 0x5b, + 0xaa, 0x0a, 0xbf, 0x0f, 0x8d, 0x36, 0x8d, 0xec, 0xd0, 0xed, 0x71, 0x23, 0xa3, 0x1a, 0xf5, 0xd6, + 0x8d, 0x41, 0xa2, 0x5d, 0x75, 0x72, 0xf0, 0xbd, 0xa0, 0xeb, 0x32, 0xda, 0xed, 0xb1, 0xbe, 0xa1, + 0x52, 0x93, 0x5d, 0x98, 0xde, 0xb5, 0x8e, 0xa8, 0x17, 0x35, 0xa7, 0x96, 0x2b, 0x77, 0x1b, 0x6b, + 0x77, 0x56, 0xd2, 0xc5, 0x5f, 0x49, 0xe7, 0xb8, 0x22, 0x08, 0x36, 0x7d, 0x16, 0xf6, 0x5b, 0x4b, + 0x83, 0x44, 0x5b, 0xf0, 0x10, 0xa0, 0x88, 0x94, 0x32, 0xc8, 0x7e, 0x6e, 0xa5, 0xe9, 0x0b, 0xad, + 0xf4, 0xc6, 0xcb, 0x44, 0x2b, 0x0d, 0x12, 0xed, 0x8a, 0xb4, 0x52, 0x2e, 0xaf, 0x60, 0x2f, 0xb2, + 0x0c, 0xe5, 0xed, 0x76, 0x73, 0x06, 0x57, 0x67, 0x61, 0x90, 0x68, 0xb3, 0xae, 0xa3, 0x0c, 0x5d, + 0xde, 0x6e, 0xdf, 0xfc, 0x00, 0x1a, 0xca, 0x1c, 0xc9, 0x02, 0x54, 0x4e, 0x69, 0x5f, 0x98, 0xd4, + 0xe0, 0x9f, 0x64, 0x09, 0xa6, 0xce, 0x2c, 0x2f, 0x96, 0x36, 0x34, 0xc4, 0x9f, 0x0f, 0xcb, 0xdf, + 0x29, 0xe9, 0x3f, 0xad, 0x42, 0xcd, 0x08, 0x84, 0x7f, 0x92, 0x77, 0x61, 0x6a, 0x9f, 0x59, 0x2c, + 0x5d, 0x8d, 0xc5, 0x41, 0xa2, 0x5d, 0x8e, 0x38, 0x40, 0x19, 0x4f, 0x50, 0x70, 0xd2, 0xbd, 0x13, + 0x2b, 0x4a, 0x57, 0x05, 0x49, 0x7b, 0x1c, 0xa0, 0x92, 0x22, 0x05, 0x79, 0x07, 0xaa, 0x8f, 0x03, + 0x87, 0xca, 0x85, 0x21, 0x83, 0x44, 0x9b, 0xef, 0x06, 0x8e, 0x4a, 0x88, 0x78, 0x72, 0x0f, 0xea, + 0x1b, 0x71, 0x18, 0x52, 0x9f, 0x6d, 0xb7, 0xd1, 0xc9, 0xea, 0xad, 0xf9, 0x41, 0xa2, 0x81, 0x2d, + 0x80, 0xdc, 0x1d, 0x73, 0x02, 0x6e, 0xea, 0x7d, 0x66, 0x85, 0x8c, 0x3a, 0xcd, 0xa9, 0x89, 0x4c, + 0xcd, 0x1d, 0xf2, 0x4a, 0x24, 0x58, 0x86, 0x4d, 0x2d, 0x25, 0x91, 0x47, 0xd0, 0x78, 0x18, 0x5a, + 0x36, 0xdd, 0xa3, 0xa1, 0x1b, 0x38, 0xb8, 0x86, 0x95, 0xd6, 0x3b, 0x83, 0x44, 0xbb, 0xd6, 0xe1, + 0x60, 0xb3, 0x87, 0xf0, 0x9c, 0xfb, 0xf3, 0x44, 0xab, 0xb5, 0xe3, 0x10, 0xad, 0x67, 0xa8, 0xac, + 0xe4, 0x4f, 0xf9, 0x92, 0x44, 0x0c, 0x4d, 0x4b, 0x1d, 0x5c, 0xbd, 0x57, 0x4f, 0x51, 0x97, 0x53, + 0xbc, 0xe6, 0x59, 0x11, 0x33, 0x43, 0xc1, 0x37, 0x34, 0x4f, 0x55, 0x24, 0x31, 0xa0, 0xb6, 0x6f, + 0x9f, 0x50, 0x27, 0xf6, 0x68, 0xb3, 0x26, 0xc5, 0x67, 0xbe, 0x9b, 0x2e, 0x69, 0x4a, 0xd1, 0xba, + 0x29, 0xc5, 0x93, 0x48, 0x42, 0x14, 0xf3, 0x67, 0x72, 0x3e, 0xac, 0x7d, 0xfa, 0xcf, 0xda, 0xa5, + 0x1f, 0xfe, 0x72, 0xf9, 0x92, 0xfe, 0xef, 0x65, 0x58, 0x18, 0x16, 0x42, 0x8e, 0x61, 0xee, 0x59, + 0xcf, 0xb1, 0x18, 0xdd, 0xf0, 0x5c, 0xea, 0xb3, 0x08, 0xfd, 0xe4, 0xd5, 0x6a, 0xfd, 0x96, 0x1c, + 0xb7, 0x19, 0x23, 0xa3, 0x69, 0x0b, 0xce, 0x21, 0xc5, 0x8a, 0x62, 0xf3, 0x71, 0x44, 0x54, 0x8b, + 0xd0, 0xc9, 0xbe, 0xd8, 0x38, 0x22, 0x38, 0x9e, 0x33, 0x8e, 0x14, 0x2b, 0x7d, 0xc8, 0x77, 0x8e, + 0xfa, 0xe8, 0x9c, 0x93, 0xfb, 0x10, 0x67, 0x19, 0xe3, 0x43, 0x1c, 0xac, 0xff, 0xba, 0x04, 0xf3, + 0x06, 0x8d, 0x82, 0x38, 0xb4, 0xe9, 0x23, 0x6a, 0x39, 0x34, 0xe4, 0x3b, 0x60, 0xc7, 0xf5, 0x1d, + 0xb9, 0xad, 0x70, 0x07, 0x9c, 0xba, 0xbe, 0xba, 0x8b, 0x11, 0x4f, 0x7e, 0x07, 0x66, 0xf6, 0xe3, + 0x23, 0x24, 0x15, 0xdb, 0xea, 0x1a, 0xae, 0x58, 0x7c, 0x64, 0x0e, 0x91, 0xa7, 0x64, 0x64, 0x15, + 0x66, 0x0e, 0x69, 0x18, 0xe5, 0x71, 0xef, 0x2a, 0x9f, 0xe1, 0x99, 0x00, 0xa9, 0x0c, 0x92, 0x8a, + 0x7c, 0x2f, 0x8f, 0xbd, 0x32, 0x90, 0x93, 0xd1, 0x88, 0x97, 0x7b, 0x4b, 0x57, 0x42, 0x54, 0x6f, + 0x49, 0xa9, 0xf4, 0xbf, 0x2a, 0x43, 0x4d, 0x98, 0xf2, 0x70, 0x8d, 0x07, 0x72, 0x45, 0x47, 0x0c, + 0xe4, 0x7c, 0xd2, 0x5f, 0x5a, 0xb3, 0xb7, 0x87, 0x35, 0x6b, 0xf0, 0x84, 0x22, 0x35, 0xcb, 0xf5, + 0xf9, 0x78, 0x22, 0x7d, 0x16, 0xa4, 0x3e, 0xb5, 0x54, 0x9f, 0x5c, 0x0b, 0xf2, 0x1d, 0xa8, 0xee, + 0xf7, 0xa8, 0x2d, 0xa3, 0xc8, 0xb5, 0x9c, 0x5b, 0xa8, 0xc6, 0x71, 0x87, 0x6b, 0xad, 0x59, 0x29, + 0xa1, 0x1a, 0xf5, 0xa8, 0x6d, 0x20, 0x87, 0xb2, 0x5b, 0x7e, 0x5e, 0x81, 0x59, 0x95, 0x9c, 0x5b, + 0x63, 0xdd, 0x71, 0x42, 0xd5, 0x1a, 0x96, 0xe3, 0x84, 0x06, 0x42, 0xc9, 0x07, 0x00, 0x7b, 0xf1, + 0x91, 0xe7, 0xda, 0x48, 0x53, 0xce, 0x13, 0x56, 0x0f, 0xa1, 0x26, 0x27, 0x55, 0x6c, 0xa2, 0x10, + 0x93, 0xbb, 0x50, 0x7b, 0x14, 0x44, 0x8c, 0xe7, 0x48, 0x69, 0x17, 0x4c, 0xd8, 0x27, 0x12, 0x66, + 0x64, 0x58, 0x62, 0x41, 0x7d, 0xa3, 0xeb, 0xc8, 0xe4, 0x56, 0xc5, 0xe4, 0xf6, 0xf6, 0x78, 0xe5, + 0x56, 0x32, 0x3a, 0x91, 0xe3, 0x6e, 0x4b, 0x5d, 0x97, 0xec, 0xae, 0x63, 0x8e, 0xe4, 0xba, 0x5c, + 0x2a, 0x77, 0xa6, 0x34, 0x46, 0x48, 0xf3, 0x91, 0xd1, 0x10, 0x94, 0x3b, 0x53, 0x28, 0x21, 0xaa, + 0x33, 0x65, 0xb9, 0xe7, 0x7d, 0xa8, 0x3f, 0x8b, 0xe8, 0x41, 0xec, 0xfb, 0xd4, 0xc3, 0xc0, 0x5b, + 0x6b, 0x35, 0xf9, 0x1c, 0xe2, 0x88, 0x9a, 0x0c, 0xa1, 0xea, 0x1c, 0x32, 0xd2, 0x9b, 0x87, 0x30, + 0x5f, 0x9c, 0xfe, 0x98, 0xf4, 0xb7, 0xa2, 0xa6, 0xbf, 0xc6, 0x5a, 0x33, 0x9f, 0xe4, 0x46, 0xd0, + 0xed, 0x5a, 0xbe, 0x60, 0x3f, 0x5c, 0x53, 0x13, 0xe3, 0x8f, 0x4a, 0x30, 0x5f, 0xc4, 0x92, 0x15, + 0x98, 0x96, 0x89, 0xa1, 0x84, 0x89, 0x81, 0xfb, 0xf0, 0xb4, 0x48, 0x09, 0x85, 0x44, 0x20, 0xa9, + 0xb8, 0x0b, 0x4b, 0x09, 0xcd, 0xf2, 0x72, 0x25, 0x75, 0x61, 0x5b, 0x80, 0x8c, 0x14, 0x47, 0x74, + 0x98, 0x36, 0x68, 0x14, 0x7b, 0x4c, 0x2e, 0x28, 0x70, 0xb1, 0x21, 0x42, 0x0c, 0x89, 0xd1, 0x7f, + 0x00, 0x70, 0xb0, 0xbb, 0xbf, 0x43, 0xfb, 0x7b, 0x96, 0x8b, 0xf1, 0x64, 0x83, 0x86, 0x0c, 0xa7, + 0x31, 0x2b, 0xe2, 0x89, 0x4d, 0x43, 0xa6, 0xc6, 0x13, 0x8e, 0x27, 0x6f, 0x41, 0x65, 0x87, 0xf6, + 0x51, 0xeb, 0xd9, 0xd6, 0x95, 0x41, 0xa2, 0xcd, 0x9d, 0x52, 0x25, 0x6e, 0x19, 0x1c, 0xab, 0xff, + 0xb4, 0x0c, 0x97, 0x39, 0xf5, 0x7a, 0xcc, 0x4e, 0x82, 0xd0, 0x65, 0xfd, 0xd7, 0x79, 0x33, 0x7f, + 0x54, 0xd8, 0xcc, 0x6f, 0x28, 0x0b, 0xad, 0x6a, 0x38, 0xd1, 0x9e, 0xfe, 0xeb, 0x2a, 0x2c, 0x8e, + 0xe1, 0x22, 0xf7, 0xa0, 0x7a, 0xd0, 0xef, 0xa5, 0x35, 0x12, 0xf7, 0xd1, 0x2a, 0x3f, 0x3c, 0x7c, + 0x9e, 0x68, 0xb3, 0x29, 0x39, 0xc7, 0x1b, 0x48, 0x45, 0xd6, 0xa0, 0xb1, 0xe1, 0xc5, 0x11, 0x93, + 0xe5, 0xbb, 0xb0, 0x17, 0x56, 0x71, 0xb6, 0x00, 0x8b, 0xfa, 0x5d, 0x25, 0x22, 0xef, 0xc1, 0xec, + 0xc6, 0x09, 0xb5, 0x4f, 0x5d, 0xbf, 0xb3, 0x43, 0xfb, 0x51, 0xb3, 0xb2, 0x5c, 0x49, 0xd7, 0xcf, + 0x96, 0x70, 0xf3, 0x94, 0xf6, 0x23, 0xa3, 0x40, 0x46, 0xbe, 0x0b, 0x8d, 0x7d, 0xb7, 0xe3, 0xa7, + 0x5c, 0x55, 0xe4, 0xba, 0xc9, 0x4b, 0x8a, 0x48, 0x80, 0x91, 0x49, 0x2d, 0x84, 0x15, 0x72, 0x5e, + 0xd0, 0x19, 0x81, 0x47, 0x45, 0x1d, 0x2c, 0x0b, 0xba, 0x90, 0x03, 0xd4, 0x82, 0x0e, 0x29, 0xc8, + 0x0e, 0xcc, 0xf0, 0x8f, 0xc7, 0x56, 0xaf, 0x39, 0x8d, 0x71, 0xe5, 0xaa, 0xba, 0xeb, 0x11, 0xd1, + 0x73, 0xfd, 0x8e, 0xba, 0xf1, 0x3d, 0x6a, 0x76, 0xad, 0x9e, 0xea, 0x1a, 0x92, 0x90, 0x7c, 0x1f, + 0x1a, 0xb9, 0x67, 0x47, 0xcd, 0x19, 0x14, 0xb8, 0x94, 0x0b, 0xcc, 0x91, 0x2d, 0x4d, 0xca, 0xbb, + 0xce, 0xbc, 0x88, 0xeb, 0x62, 0xf6, 0x38, 0x8b, 0xaa, 0x90, 0x22, 0xa9, 0x10, 0x9c, 0x6a, 0xaf, + 0x0c, 0x4e, 0xa5, 0x8b, 0x82, 0x93, 0x6e, 0x40, 0x43, 0x51, 0x4c, 0xec, 0xd8, 0x6e, 0x90, 0x15, + 0xca, 0x72, 0xc7, 0x72, 0x88, 0x21, 0x31, 0x44, 0x83, 0xa9, 0xdd, 0xc0, 0xb6, 0x3c, 0xb9, 0xf5, + 0xeb, 0x83, 0x44, 0x9b, 0xf2, 0x38, 0xc0, 0x10, 0x70, 0xfd, 0xbf, 0x4a, 0xb0, 0xb0, 0x17, 0x06, + 0x67, 0x2e, 0x77, 0xfd, 0x83, 0xe0, 0x94, 0xfa, 0x87, 0xdf, 0x26, 0xdb, 0xe9, 0x2a, 0x94, 0x90, + 0xeb, 0x3e, 0xe7, 0xc2, 0x55, 0xf8, 0x3c, 0xd1, 0xde, 0xb9, 0xf0, 0x54, 0x89, 0xd6, 0x4f, 0x57, + 0x49, 0x39, 0x8b, 0x94, 0x27, 0x2f, 0x6e, 0x2e, 0x38, 0x8b, 0x68, 0x30, 0x85, 0x53, 0x95, 0xdb, + 0x18, 0xb5, 0x62, 0x1c, 0x60, 0x08, 0xb8, 0xb2, 0x7f, 0xfe, 0xa1, 0x3c, 0xa2, 0xdf, 0x6b, 0x1c, + 0x58, 0x3e, 0x2e, 0x04, 0x16, 0xe5, 0x94, 0x58, 0x54, 0x71, 0xa2, 0xc8, 0x62, 0xc1, 0xd2, 0x38, + 0xae, 0xaf, 0x70, 0xf1, 0xf5, 0xbf, 0x2f, 0xc3, 0x3c, 0x3f, 0xa8, 0xb9, 0x36, 0x0e, 0x10, 0xbd, + 0xce, 0xa6, 0xff, 0x83, 0x82, 0xe9, 0x6f, 0x2b, 0x35, 0x8c, 0xa2, 0xe0, 0x44, 0x86, 0x3f, 0x05, + 0x32, 0xca, 0x43, 0x9e, 0xc1, 0xac, 0x0a, 0x45, 0xeb, 0x17, 0x0e, 0x53, 0xc3, 0xbb, 0xb4, 0x75, + 0x55, 0x8e, 0x32, 0x17, 0x21, 0x9f, 0x89, 0x3b, 0x20, 0x32, 0x0a, 0x62, 0xf4, 0xbf, 0x2b, 0xc3, + 0x9c, 0x12, 0xd5, 0x5f, 0xe7, 0x15, 0x78, 0x50, 0x58, 0x81, 0x5b, 0x4a, 0x56, 0xcd, 0xf5, 0x9b, + 0x68, 0x01, 0x1e, 0xc2, 0x95, 0x11, 0x96, 0xe1, 0x14, 0x59, 0x9a, 0x20, 0x45, 0x8a, 0xa2, 0x45, + 0xfc, 0xdf, 0x08, 0xfc, 0x63, 0xb7, 0x73, 0x78, 0xff, 0xeb, 0x58, 0xb4, 0xa8, 0x1a, 0xa2, 0xb5, + 0xee, 0x5f, 0x60, 0xe0, 0x9f, 0x4c, 0xc1, 0xe2, 0x18, 0x2e, 0xb2, 0x0e, 0x0b, 0xfb, 0x34, 0xc2, + 0x89, 0x53, 0x3b, 0x08, 0x1d, 0xd7, 0xef, 0x48, 0x3b, 0xe1, 0x81, 0x31, 0x12, 0x38, 0x33, 0x4c, + 0x91, 0xc6, 0x08, 0x39, 0xb6, 0x67, 0x84, 0xe4, 0xed, 0xb6, 0x34, 0xa1, 0x68, 0xcf, 0xc8, 0x45, + 0xc2, 0xf6, 0x4c, 0x4a, 0x40, 0x76, 0x61, 0x71, 0x2f, 0x0c, 0x5e, 0xf4, 0xb1, 0x42, 0x89, 0xf8, + 0xa1, 0x44, 0x96, 0x32, 0x9c, 0x0f, 0x8b, 0x92, 0x1e, 0x47, 0x9b, 0x58, 0xd0, 0x44, 0x26, 0x3f, + 0xbf, 0x88, 0x9a, 0x66, 0x1c, 0x1b, 0xf9, 0x10, 0xa6, 0xd6, 0x63, 0xc7, 0x65, 0xd2, 0xc0, 0x4a, + 0xbd, 0x81, 0x60, 0xa1, 0x6a, 0x6b, 0x4e, 0x9a, 0x66, 0xca, 0xe2, 0x40, 0x43, 0xb0, 0x90, 0x4f, + 0xb8, 0xcf, 0xb9, 0xd4, 0x67, 0xdb, 0x8e, 0x47, 0x79, 0xc6, 0x0b, 0x62, 0x86, 0xa6, 0xae, 0xb4, + 0xde, 0x1a, 0x24, 0xda, 0xa2, 0xe8, 0x48, 0x98, 0xae, 0xe3, 0x51, 0x93, 0x09, 0x74, 0xa1, 0x9a, + 0x1f, 0xe5, 0x26, 0x3f, 0x80, 0xab, 0x6d, 0x37, 0xb2, 0x03, 0xdf, 0xa7, 0x36, 0x13, 0xa9, 0xd1, + 0xc1, 0x82, 0x5c, 0x9c, 0x5b, 0xb8, 0xd8, 0xeb, 0x4e, 0x46, 0x60, 0x8a, 0x9c, 0xea, 0x98, 0xbc, + 0x46, 0xff, 0x3c, 0xd1, 0xaa, 0xad, 0x20, 0xf0, 0x8c, 0xf1, 0x12, 0xf8, 0x6c, 0xb3, 0xd6, 0xef, + 0xb6, 0xcf, 0x68, 0x78, 0x66, 0x79, 0xb2, 0xf7, 0x87, 0xb3, 0x3d, 0xa5, 0xb4, 0x67, 0x5a, 0x1c, + 0x6b, 0xba, 0x12, 0x5d, 0x9c, 0xed, 0x08, 0x37, 0xd9, 0x52, 0x44, 0x6e, 0x04, 0xb1, 0xcf, 0x1e, + 0x5b, 0x2f, 0xb0, 0x22, 0xaa, 0x88, 0x13, 0x96, 0x22, 0xd2, 0xe6, 0x68, 0xb3, 0x6b, 0xbd, 0x30, + 0x46, 0x59, 0xc8, 0xef, 0x42, 0x1d, 0x2b, 0x17, 0x5e, 0xe1, 0x36, 0xeb, 0xa8, 0x29, 0xdf, 0x43, + 0x80, 0x55, 0x8d, 0x69, 0xc5, 0xec, 0x24, 0x53, 0x2e, 0x27, 0xd4, 0x3f, 0xad, 0x40, 0x43, 0x59, + 0x24, 0x7e, 0x76, 0x51, 0xca, 0x67, 0x3c, 0xbb, 0xf0, 0xf2, 0x59, 0x3d, 0xbb, 0x60, 0xe1, 0x7c, + 0x8f, 0xd7, 0x58, 0x1d, 0xbe, 0xf9, 0x84, 0xaf, 0x61, 0xe3, 0x35, 0x44, 0x88, 0xda, 0x78, 0x15, + 0x34, 0x64, 0x17, 0x16, 0x70, 0x10, 0xe9, 0xb5, 0xd1, 0x33, 0x63, 0x5b, 0xfa, 0xda, 0xf2, 0x20, + 0xd1, 0x6e, 0xa3, 0x43, 0x98, 0xd2, 0xcb, 0x23, 0x33, 0x0e, 0x5d, 0x45, 0xc6, 0x08, 0x27, 0xf9, + 0xa7, 0x12, 0xcc, 0x23, 0x70, 0xf3, 0x8c, 0xfa, 0x0c, 0x85, 0x55, 0x65, 0x77, 0x20, 0x6b, 0xd5, + 0xef, 0xb3, 0xd0, 0xf5, 0x3b, 0x87, 0xfc, 0xbc, 0x18, 0xb5, 0xfe, 0x88, 0x7b, 0xde, 0x2f, 0x12, + 0xed, 0xfd, 0x2f, 0xd6, 0xf8, 0x97, 0x42, 0xa2, 0x41, 0xa2, 0xdd, 0x14, 0x53, 0xa4, 0x38, 0xe0, + 0xd0, 0x04, 0x87, 0xe6, 0x42, 0xb6, 0xe4, 0xec, 0x0e, 0xac, 0x23, 0x8f, 0x62, 0xcc, 0x9c, 0x42, + 0x55, 0xef, 0xe4, 0x72, 0x18, 0x47, 0x61, 0xdc, 0x1c, 0x91, 0x93, 0x71, 0xe9, 0xff, 0x57, 0x52, + 0xda, 0xeb, 0xaf, 0x6f, 0xf8, 0xfc, 0xa0, 0x10, 0x3e, 0xaf, 0xe7, 0xdc, 0x99, 0x6e, 0x1c, 0x3d, + 0x2e, 0x70, 0xea, 0x97, 0x61, 0xae, 0x40, 0x84, 0x79, 0x65, 0xdd, 0xb6, 0x69, 0x14, 0x19, 0xf4, + 0xcf, 0x62, 0x1a, 0xb1, 0xaf, 0x65, 0x5e, 0x29, 0x68, 0x38, 0x51, 0x5e, 0xf9, 0xcf, 0x32, 0x2c, + 0x8e, 0xe1, 0xe2, 0xb6, 0x79, 0x16, 0xd1, 0x42, 0x9f, 0x2b, 0x8e, 0x68, 0x68, 0x20, 0x94, 0x9f, + 0x16, 0x44, 0x41, 0xab, 0x9c, 0x81, 0xb0, 0xa0, 0x4d, 0xcf, 0x28, 0xeb, 0xe9, 0x85, 0x03, 0x37, + 0xc4, 0xbc, 0xda, 0x7c, 0x4b, 0x87, 0xe1, 0xd8, 0x57, 0x5e, 0x44, 0xec, 0xc3, 0xcc, 0x46, 0x48, + 0xb1, 0xc9, 0x5e, 0x9d, 0xfc, 0x98, 0x63, 0x0b, 0x96, 0xe1, 0x63, 0x8e, 0x94, 0xa4, 0x9e, 0x9d, + 0xa6, 0xbe, 0xaa, 0xb3, 0x93, 0xfe, 0x97, 0xa5, 0x21, 0x1b, 0x6e, 0xb9, 0x1e, 0xa3, 0x21, 0xb9, + 0x86, 0xf7, 0x3b, 0xc2, 0x82, 0xd3, 0x83, 0x44, 0x2b, 0xbb, 0x8e, 0x51, 0xde, 0x6e, 0x67, 0xb6, + 0x2d, 0x8f, 0xb5, 0xed, 0xef, 0x4d, 0x66, 0x3a, 0xb4, 0x39, 0x9a, 0x4e, 0x1a, 0x4c, 0xff, 0x8b, + 0x32, 0x4c, 0x73, 0xeb, 0xbf, 0xce, 0x9e, 0xfd, 0x7e, 0xc1, 0xb3, 0x97, 0x8a, 0xed, 0x87, 0x89, + 0x1c, 0xfa, 0xd7, 0x25, 0x80, 0x9c, 0x98, 0x7c, 0x0f, 0x66, 0x9e, 0xe2, 0x85, 0x60, 0x7a, 0xa7, + 0x31, 0xd4, 0xd2, 0x90, 0xc8, 0xd6, 0x8d, 0x74, 0xad, 0x03, 0x01, 0x50, 0xad, 0x20, 0x69, 0xc8, + 0x43, 0x98, 0x5a, 0xf7, 0xbc, 0xe0, 0xf9, 0x68, 0xb7, 0x91, 0x4b, 0xda, 0x08, 0x7c, 0xc7, 0x15, + 0xc2, 0xae, 0x4b, 0x61, 0x97, 0x2d, 0x4e, 0xae, 0xba, 0x36, 0xf2, 0x93, 0x36, 0x54, 0xdb, 0xd4, + 0x4f, 0xef, 0x26, 0xce, 0x97, 0x73, 0x4d, 0xca, 0x99, 0x77, 0xa8, 0xaf, 0xb6, 0xf7, 0x90, 0x5b, + 0xff, 0x59, 0x55, 0x34, 0x2f, 0xd2, 0xe9, 0x3d, 0x80, 0xd9, 0xad, 0x20, 0x7c, 0x6e, 0x85, 0xce, + 0x7a, 0x87, 0xfa, 0xa2, 0x89, 0x58, 0xc3, 0xf6, 0xf3, 0xdc, 0xb1, 0x80, 0x9b, 0x16, 0x47, 0x64, + 0xc9, 0xbc, 0x40, 0x4e, 0x9e, 0xc2, 0xdc, 0x63, 0xeb, 0x85, 0xcc, 0x96, 0x07, 0x07, 0xbb, 0xa8, + 0x65, 0xa5, 0xf5, 0xee, 0x20, 0xd1, 0x6e, 0x74, 0xad, 0x17, 0x69, 0x92, 0x35, 0x19, 0xf3, 0xce, + 0xb9, 0x27, 0x2b, 0xf2, 0x13, 0x0f, 0xe6, 0xf7, 0x82, 0x90, 0xc9, 0x41, 0x78, 0x61, 0x2a, 0xf4, + 0x5d, 0xcc, 0xf5, 0xe5, 0xd3, 0xc0, 0x4c, 0xdb, 0x5a, 0x7d, 0x99, 0x68, 0xa5, 0x5f, 0x24, 0x1a, + 0x70, 0x90, 0xd0, 0x88, 0x0f, 0xcc, 0x33, 0xab, 0x79, 0x9c, 0x49, 0x50, 0x73, 0x5e, 0x51, 0x36, + 0x79, 0x00, 0x57, 0x78, 0x9d, 0xe5, 0x1e, 0xbb, 0xb6, 0xc5, 0xe8, 0x56, 0x10, 0x76, 0x2d, 0x26, + 0x2f, 0x1b, 0xf1, 0x52, 0x9d, 0xd7, 0x68, 0x5c, 0x52, 0xd7, 0x62, 0xc6, 0x28, 0x25, 0xf9, 0xc3, + 0xf3, 0x8b, 0xc9, 0x6f, 0x0d, 0x12, 0xed, 0x8d, 0x31, 0xc5, 0xe4, 0x39, 0x56, 0x18, 0x53, 0x56, + 0x76, 0x5e, 0x5d, 0x56, 0x7e, 0x5b, 0xb6, 0xaa, 0xde, 0x3c, 0xa7, 0xb4, 0x2c, 0x0c, 0xf4, 0xca, + 0x22, 0x73, 0x0d, 0x2a, 0xad, 0xbd, 0x2d, 0xec, 0xb5, 0xc9, 0x02, 0x89, 0xfa, 0x27, 0x96, 0x6f, + 0x53, 0x27, 0x3f, 0x01, 0xa8, 0x6d, 0xe2, 0xd6, 0xde, 0x96, 0xfe, 0x93, 0x0a, 0xcc, 0x17, 0xfd, + 0x8e, 0xe8, 0x30, 0xbd, 0x1b, 0x74, 0x5c, 0x3f, 0x6d, 0x58, 0x60, 0x1b, 0xcc, 0x43, 0x88, 0x21, + 0x31, 0xe4, 0x6d, 0x80, 0x2c, 0xc3, 0xa6, 0x79, 0x40, 0x5e, 0xe1, 0x2b, 0x08, 0xf2, 0x27, 0x00, + 0x4f, 0x02, 0x87, 0xca, 0xdb, 0x8a, 0x8a, 0xdc, 0x82, 0x59, 0x9d, 0x24, 0x3a, 0xf0, 0xa2, 0xd6, + 0xfa, 0x86, 0xac, 0xb5, 0xe4, 0x9d, 0xfb, 0x20, 0xd1, 0xae, 0xfa, 0x81, 0x43, 0x47, 0x2f, 0x2a, + 0x14, 0x89, 0xe4, 0x01, 0x4c, 0x19, 0xb1, 0x47, 0xd3, 0x8b, 0x90, 0x79, 0x65, 0x2f, 0xc5, 0x1e, + 0xcd, 0x77, 0x62, 0x18, 0x0f, 0x77, 0x3c, 0x39, 0x80, 0x7c, 0x04, 0xb0, 0x13, 0x1f, 0xd1, 0x87, + 0x61, 0x10, 0xf7, 0xd2, 0x0e, 0xa9, 0x36, 0x48, 0xb4, 0x5b, 0xa7, 0xf1, 0x11, 0x0d, 0x7d, 0xca, + 0x68, 0x64, 0x76, 0x10, 0xa9, 0x8e, 0x9f, 0xb3, 0x10, 0x03, 0x66, 0x64, 0x58, 0x96, 0x0f, 0x03, + 0xde, 0x3c, 0x27, 0x1b, 0x2b, 0xdb, 0x1a, 0x4f, 0x66, 0xa1, 0x00, 0x17, 0x3a, 0xa7, 0x02, 0xa4, + 0xb7, 0xe1, 0xfa, 0x39, 0xac, 0x79, 0x33, 0xb7, 0x74, 0x51, 0x33, 0x57, 0xff, 0x59, 0x09, 0xaa, + 0x5c, 0x49, 0xf2, 0x1e, 0xd4, 0xd3, 0x6b, 0xcb, 0x94, 0xef, 0x3a, 0x3f, 0x71, 0x84, 0x29, 0x50, + 0xbd, 0x7f, 0xc9, 0x28, 0xf9, 0x50, 0x87, 0x34, 0x3c, 0x4a, 0xd7, 0x16, 0x87, 0x3a, 0xe3, 0x00, + 0x75, 0x28, 0xa4, 0xe0, 0xa4, 0xdf, 0x3f, 0xa1, 0x61, 0x7a, 0x71, 0x85, 0xa4, 0xcf, 0x39, 0x40, + 0x25, 0x45, 0x0a, 0xb2, 0x0a, 0x33, 0xeb, 0xb6, 0x88, 0xc7, 0x55, 0x94, 0x8b, 0xc6, 0xb0, 0xec, + 0x91, 0xa0, 0x2b, 0xa9, 0xf4, 0x37, 0xa1, 0x9e, 0x45, 0x09, 0xb2, 0x04, 0x53, 0xf8, 0x21, 0x62, + 0x9b, 0x21, 0xfe, 0x60, 0xe2, 0xe3, 0xa9, 0xf3, 0x75, 0xee, 0xc4, 0x9c, 0x9b, 0xf8, 0xb8, 0x62, + 0x13, 0xb5, 0x60, 0xfe, 0x6d, 0x1a, 0x20, 0x27, 0x26, 0x14, 0xe6, 0x9f, 0x6e, 0xb7, 0x37, 0xb6, + 0x1d, 0xea, 0x33, 0x97, 0xb9, 0x74, 0x4c, 0xfb, 0x6b, 0xf3, 0x05, 0xa3, 0xa1, 0x6f, 0x79, 0x92, + 0xa6, 0xdf, 0x7a, 0x53, 0x0e, 0x70, 0x23, 0x70, 0x1d, 0xdb, 0x74, 0x33, 0x56, 0x35, 0xec, 0x16, + 0x85, 0xf2, 0x61, 0xf6, 0xd7, 0x1f, 0xef, 0x2a, 0xc3, 0x94, 0x27, 0x1f, 0x26, 0xb2, 0xba, 0xde, + 0x39, 0xc3, 0x14, 0x85, 0x92, 0x53, 0x58, 0x78, 0x88, 0x27, 0x30, 0x65, 0xa0, 0xca, 0x85, 0x03, + 0xbd, 0x25, 0x07, 0xba, 0x25, 0x4e, 0x6f, 0xe3, 0x87, 0x1a, 0x11, 0x9c, 0x6f, 0xb2, 0xea, 0x85, + 0x37, 0x26, 0x3f, 0x2c, 0xc1, 0xf4, 0x41, 0x68, 0xb9, 0x2c, 0xad, 0x27, 0xcf, 0x89, 0x6d, 0x9f, + 0xc8, 0xd8, 0xf6, 0xde, 0x17, 0x3c, 0x47, 0x0a, 0xd9, 0xfc, 0x84, 0xcc, 0xf0, 0x4b, 0x3d, 0x21, + 0x0b, 0x1c, 0x79, 0x08, 0xd3, 0xbc, 0xfe, 0x8b, 0xd3, 0x97, 0x49, 0x4a, 0x81, 0x83, 0xa1, 0x5a, + 0x20, 0x5b, 0x4d, 0x69, 0x8b, 0x85, 0x08, 0xff, 0xab, 0x82, 0x04, 0x85, 0xfa, 0x12, 0x6c, 0xe6, + 0x37, 0x7b, 0x09, 0x46, 0x9e, 0x42, 0x5d, 0x56, 0xdc, 0xad, 0xbe, 0xbc, 0xa3, 0x51, 0xb2, 0x7e, + 0x86, 0x52, 0x2e, 0xa4, 0x05, 0xc8, 0x54, 0x5f, 0x5f, 0x18, 0xb9, 0x0c, 0x62, 0x0c, 0xb7, 0x28, + 0x0a, 0x0b, 0x9f, 0xa1, 0xf6, 0xa9, 0x1d, 0x52, 0x16, 0x89, 0xf6, 0x47, 0xde, 0xbe, 0x50, 0x65, + 0xe6, 0x0d, 0x8c, 0x1f, 0x97, 0x60, 0x61, 0xd8, 0x65, 0xc8, 0x77, 0xa1, 0xb1, 0x21, 0xf2, 0x6a, + 0x10, 0x66, 0xc5, 0x3b, 0xb6, 0xb5, 0xec, 0x14, 0x6c, 0x16, 0x9e, 0x69, 0xa9, 0xe4, 0x64, 0x0d, + 0x6a, 0x7c, 0x0b, 0xfa, 0xf9, 0x8d, 0x20, 0x46, 0x98, 0x58, 0xc2, 0xd4, 0x2b, 0xa8, 0x94, 0x4e, + 0xd9, 0xc1, 0xff, 0x51, 0x86, 0x86, 0xb2, 0x64, 0xe4, 0x5d, 0xa8, 0x6d, 0x47, 0xbb, 0x81, 0x7d, + 0x4a, 0x1d, 0x59, 0xcc, 0xe1, 0x73, 0x3f, 0x37, 0x32, 0x3d, 0x04, 0x1a, 0x19, 0x9a, 0xb4, 0x60, + 0x4e, 0x7c, 0x3d, 0xa6, 0x51, 0x64, 0x75, 0xd2, 0xd1, 0x6f, 0x0f, 0x12, 0xad, 0x29, 0x88, 0xcd, + 0xae, 0xc0, 0x28, 0x73, 0x28, 0xb2, 0x90, 0x3f, 0x06, 0x10, 0x00, 0xbe, 0xca, 0x13, 0xbc, 0x9b, + 0x49, 0xb7, 0xf1, 0x55, 0x39, 0x00, 0x2f, 0x8b, 0x86, 0x8e, 0x48, 0x8a, 0x40, 0x7c, 0x38, 0x15, + 0xd8, 0xa7, 0x93, 0x3f, 0x36, 0xcc, 0x1f, 0x4e, 0x05, 0xf6, 0xa9, 0x39, 0xfe, 0x0c, 0xa6, 0x8a, + 0xd4, 0x7f, 0x55, 0x52, 0xdc, 0x8e, 0x7c, 0x02, 0xf5, 0x6c, 0x69, 0x64, 0xed, 0x7f, 0x4d, 0x7d, + 0x1f, 0x20, 0x51, 0x06, 0x3d, 0x6e, 0xdd, 0x92, 0x05, 0xd8, 0x62, 0xb6, 0xc6, 0x05, 0x2f, 0x4c, + 0x81, 0xe4, 0x63, 0xa8, 0xa2, 0x6d, 0x2e, 0xbe, 0x76, 0x4b, 0x43, 0x7d, 0x95, 0x1b, 0x05, 0x67, + 0x8a, 0x9c, 0xe4, 0xbe, 0x3c, 0xfa, 0x09, 0xeb, 0x5e, 0x29, 0x86, 0x79, 0x3e, 0x95, 0x2c, 0xc6, + 0xe7, 0x27, 0x42, 0xc5, 0x43, 0xfe, 0xbc, 0x04, 0x8b, 0xcf, 0xd6, 0xb6, 0x0c, 0xda, 0x71, 0x23, + 0x26, 0xea, 0xcd, 0x36, 0xcf, 0x1e, 0x37, 0xa0, 0x62, 0x58, 0xcf, 0xe5, 0xb3, 0x81, 0x99, 0x41, + 0xa2, 0x55, 0x42, 0xeb, 0xb9, 0xc1, 0x61, 0xe4, 0x1e, 0xd4, 0x77, 0x68, 0xff, 0x91, 0xe5, 0x3b, + 0x1e, 0x95, 0x0f, 0x06, 0xb0, 0xbb, 0x7b, 0x4a, 0xfb, 0xe6, 0x09, 0x42, 0x8d, 0x9c, 0x80, 0x57, + 0x7e, 0x7b, 0xf1, 0xd1, 0x0e, 0x15, 0x67, 0x93, 0x59, 0x51, 0xf9, 0xf5, 0xe2, 0xa3, 0x53, 0xda, + 0x37, 0x24, 0x46, 0xff, 0x97, 0x32, 0x2c, 0x0c, 0xef, 0x38, 0xf2, 0x11, 0xcc, 0xee, 0x59, 0x51, + 0xf4, 0x3c, 0x08, 0x9d, 0x47, 0x56, 0x74, 0x22, 0xa7, 0x72, 0x6b, 0x90, 0x68, 0xd7, 0x7b, 0x12, + 0x6e, 0x9e, 0x58, 0x91, 0xba, 0x15, 0x0b, 0x0c, 0x3c, 0x37, 0x1f, 0x3c, 0x3d, 0xd8, 0x4b, 0x9f, + 0x35, 0xc8, 0x9d, 0xc3, 0x02, 0xd6, 0x33, 0x8b, 0x6f, 0x1b, 0x52, 0x32, 0xd2, 0x81, 0xcb, 0x43, + 0xb6, 0x90, 0x66, 0x55, 0x1a, 0x22, 0x63, 0x8c, 0x25, 0xba, 0x69, 0xf1, 0xda, 0xb1, 0x19, 0x2a, + 0x18, 0x65, 0x80, 0x61, 0xa9, 0xe4, 0x03, 0x80, 0x67, 0x6b, 0x5b, 0xd8, 0x2e, 0xa5, 0x21, 0x3a, + 0xee, 0x9c, 0x78, 0xd5, 0xc3, 0x85, 0xd8, 0x02, 0xac, 0x96, 0x87, 0x39, 0xb1, 0xee, 0xc3, 0xac, + 0xea, 0x69, 0xbc, 0x3e, 0x51, 0x9a, 0xa4, 0xb5, 0xf4, 0x8d, 0x81, 0x6c, 0x8d, 0x8a, 0x86, 0x41, + 0x79, 0xa4, 0x61, 0x70, 0x17, 0x6a, 0x69, 0x80, 0x52, 0xdf, 0x06, 0xc9, 0x74, 0xd6, 0x37, 0x32, + 0xac, 0xfe, 0x0d, 0x98, 0x91, 0x9e, 0xf4, 0xea, 0x07, 0xb8, 0xfa, 0x8f, 0xca, 0x70, 0xd9, 0xa0, + 0xbc, 0x90, 0x91, 0xef, 0x6d, 0xbe, 0x96, 0x8f, 0x43, 0x0a, 0x1a, 0x9e, 0x5f, 0x45, 0xe9, 0xff, + 0x5a, 0x82, 0xc5, 0x31, 0xb4, 0x5f, 0xe6, 0x06, 0x8b, 0xbc, 0x0f, 0xf5, 0xb6, 0x6b, 0x79, 0xeb, + 0x8e, 0x13, 0xa6, 0xb5, 0x33, 0xa6, 0x23, 0xc7, 0xe5, 0xd9, 0x88, 0x43, 0xd5, 0xe0, 0x92, 0x91, + 0x92, 0x6f, 0x4a, 0xd7, 0xa8, 0x64, 0xc6, 0x4d, 0x9f, 0x9f, 0x80, 0x98, 0x53, 0xfe, 0xf8, 0x44, + 0xff, 0xc7, 0x32, 0x10, 0x01, 0x94, 0xde, 0xe5, 0x06, 0xaf, 0xf5, 0x25, 0x7c, 0xab, 0xb0, 0x80, + 0xcb, 0xca, 0x23, 0x91, 0x21, 0x25, 0x27, 0xaa, 0x84, 0x7f, 0x5c, 0x86, 0x6b, 0xe3, 0x19, 0xbf, + 0xd4, 0x82, 0xde, 0x83, 0x3a, 0x5e, 0x5d, 0x29, 0xef, 0x7c, 0x30, 0x82, 0x8a, 0x7b, 0x2e, 0xa4, + 0xcf, 0x09, 0xc8, 0x31, 0xcc, 0xed, 0x5a, 0x11, 0x7b, 0x44, 0xad, 0x90, 0x1d, 0x51, 0x8b, 0x4d, + 0x90, 0x48, 0xb3, 0x27, 0xae, 0xf8, 0x42, 0xf8, 0x24, 0xe5, 0x1c, 0x7e, 0xe2, 0x5a, 0x10, 0x9b, + 0xb9, 0x4b, 0xf5, 0x62, 0x77, 0xf9, 0xe6, 0x47, 0x30, 0xab, 0xf6, 0x0e, 0x49, 0x0d, 0xaa, 0x4f, + 0x9e, 0x3e, 0xd9, 0x5c, 0xb8, 0x44, 0x1a, 0x30, 0xb3, 0xb7, 0xf9, 0xa4, 0xbd, 0xfd, 0xe4, 0xe1, + 0x42, 0x89, 0xcc, 0x42, 0x6d, 0x7d, 0x6f, 0xcf, 0x78, 0x7a, 0xb8, 0xd9, 0x5e, 0x28, 0x13, 0x80, + 0xe9, 0xf6, 0xe6, 0x93, 0xed, 0xcd, 0xf6, 0x42, 0xa5, 0xb5, 0xf4, 0xf2, 0x7f, 0xee, 0x5c, 0x7a, + 0xf9, 0xd9, 0x9d, 0xd2, 0xcf, 0x3f, 0xbb, 0x53, 0xfa, 0xef, 0xcf, 0xee, 0x94, 0x3e, 0xfd, 0xdf, + 0x3b, 0x97, 0x8e, 0xa6, 0x51, 0x97, 0xfb, 0xff, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x00, 0x25, 0x85, + 0x86, 0x64, 0x31, 0x00, 0x00, } diff --git a/lib/services/types.proto b/lib/services/types.proto index 0badccfdb9e..ac8cc001de5 100644 --- a/lib/services/types.proto +++ b/lib/services/types.proto @@ -471,6 +471,9 @@ message RoleOptions { // DisconnectExpiredCert sets disconnect clients on expired certificates. bool DisconnectExpiredCert = 6 [(gogoproto.nullable) = true, (gogoproto.jsontag) = "disconnect_expired_cert,omitempty", (gogoproto.casttype) = "Bool"]; + + // BPF defines what events to record for the BPF-based session recorder. + repeated string BPF = 7 [(gogoproto.jsontag) = "enhanced_recording,omitempty"]; } diff --git a/lib/srv/ctx.go b/lib/srv/ctx.go index 3651be97ac7..9af3c7b1a45 100644 --- a/lib/srv/ctx.go +++ b/lib/srv/ctx.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "net" + "os" "sync" "sync/atomic" "time" @@ -30,6 +31,7 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/pam" @@ -115,6 +117,9 @@ type Server interface { // UseTunnel used to determine if this node has connected to this cluster // using reverse tunnel. UseTunnel() bool + + // GetBPF returns the BPF service used for enhanced session recording. + GetBPF() bpf.BPF } // IdentityContext holds all identity information associated with the user @@ -250,6 +255,24 @@ type ServerContext struct { // cancel is called whenever server context is closed cancel context.CancelFunc + + // termAllocated is used to track if a terminal has been allocated. This has + // to be tracked because the terminal is set to nil after it's "taken" in the + // session. Terminals can be allocated for both "exec" or "session" requests. + termAllocated bool + + // request is the request that was issued by the client + request *ssh.Request + + // cmd{r,w} are used to send the command from the parent process to the + // child process. + cmdr *os.File + cmdw *os.File + + // cont{r,w} is used to send the continue signal from the parent process + // to the child process. + contr *os.File + contw *os.File } // NewServerContext creates a new *ServerContext which is used to pass and @@ -320,6 +343,23 @@ func NewServerContext(srv Server, conn *ssh.ServerConn, identityContext Identity } go mon.Start() } + + // Create pipe used to send command to child process. + ctx.cmdr, ctx.cmdw, err = os.Pipe() + if err != nil { + return nil, trace.Wrap(err) + } + ctx.AddCloser(ctx.cmdr) + ctx.AddCloser(ctx.cmdw) + + // Create pipe used to signal continue to child process. + ctx.contr, ctx.contw, err = os.Pipe() + if err != nil { + return nil, trace.Wrap(err) + } + ctx.AddCloser(ctx.contr) + ctx.AddCloser(ctx.contw) + return ctx, nil } diff --git a/lib/srv/exec.go b/lib/srv/exec.go index 9e1ef39486c..d088dfeb822 100644 --- a/lib/srv/exec.go +++ b/lib/srv/exec.go @@ -18,8 +18,12 @@ package srv import ( "bufio" + "bytes" + "context" + "encoding/json" "fmt" "io" + "io/ioutil" "net" "os" "os/exec" @@ -28,17 +32,20 @@ import ( "strconv" "strings" "syscall" + "time" "golang.org/x/crypto/ssh" "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/events" + "github.com/gravitational/teleport/lib/pam" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/shell" + "github.com/gravitational/teleport/lib/sshutils" "github.com/gravitational/teleport/lib/utils" "github.com/gravitational/trace" - "github.com/kardianos/osext" log "github.com/sirupsen/logrus" ) @@ -49,6 +56,55 @@ const ( defaultLoginDefsPath = "/etc/login.defs" ) +// execCommand contains the payload to "teleport exec" will will be used to +// construct and execute a exec.Cmd. +type execCommand struct { + // Path the the full path to the binary to execute. + Path string `json:"path"` + + // Args is the list of arguments to pass to the command. + Args []string `json:"args"` + + // Env is a list of environment variables to pass to the command. + Env []string `json:"env"` + + // Dir is the working/home directory of the command. + Dir string `json:"dir"` + + // Uid is the UID under which to spawn the command. + Uid uint32 `json:"uid"` + + // Gid it the GID under which to spawn the command. + Gid uint32 `json:"gid"` + + // Groups is the list of supplementary groups. + Groups []uint32 `json:"groups"` + + // SetCreds controls if the process credentials will be set. + SetCreds bool `json:"set_creds"` + + // Terminal is if a TTY has been allocated for the session. + Terminal bool `json:"term"` + + // RequestType is the type of request: either "exec" or "shell". + RequestType string `json:"request_type"` + + // PAM contains metadata needed to launch a PAM context. + PAM *pamCommand `json:"pam"` +} + +// pamCommand contains the payload to launch a PAM context. +type pamCommand struct { + // Enabled indicates that PAM has been enabled on this host. + Enabled bool `json:"enabled"` + + // ServiceName is the name service whose policy will be loaded. + ServiceName string `json:"service_name"` + + // Username is the host login. + Username string `json:"username"` +} + // ExecResult is used internally to send the result of a command execution from // a goroutine to SSH request handler and back to the calling client type ExecResult struct { @@ -72,6 +128,13 @@ type Exec interface { // Wait will block while the command executes. Wait() *ExecResult + + // Continue will resume execution of the process after it completes its + // pre-processing routine (placed in a cgroup). + Continue() + + // PID returns the PID of the Teleport process that was re-execed. + PID() int } // NewExecRequest creates a new local or remote Exec. @@ -114,6 +177,10 @@ type localExec struct { // Ctx holds the *ServerContext. Ctx *ServerContext + + // sessionContext holds the BPF session context used to lookup and interact + // with BPF sessions. + sessionContext *bpf.SessionContext } // GetCommand returns the command string. @@ -129,35 +196,36 @@ func (e *localExec) SetCommand(command string) { // Start launches the given command returns (nil, nil) if successful. // ExecResult is only used to communicate an error while launching. func (e *localExec) Start(channel ssh.Channel) (*ExecResult, error) { - var err error - - // parse the command to see if the user is trying to run scp - err = e.transformSecureCopy() + // Parse the command to see if it is scp. + err := e.transformSecureCopy() if err != nil { return nil, trace.Wrap(err) } - // transforms the Command string into *exec.Cmd - e.Cmd, err = prepareCommand(e.Ctx) + // Create the command that will actually execute. + e.Cmd, err = configureCommand(e.Ctx) if err != nil { return nil, trace.Wrap(err) } - // hook up stdout/err the channel so the user can interact with the command + // Connect stdout and stderr to the channel so the user can interact with + // the command. e.Cmd.Stderr = channel.Stderr() e.Cmd.Stdout = channel + + // Copy from the channel (client input) into stdin of the process. inputWriter, err := e.Cmd.StdinPipe() if err != nil { return nil, trace.Wrap(err) } - go func() { - // copy from the channel (client) into stdin of the process io.Copy(inputWriter, channel) inputWriter.Close() }() - if err := e.Cmd.Start(); err != nil { + // Start the command. + err = e.Cmd.Start() + if err != nil { e.Ctx.Warningf("Local command %v failed to start: %v", e.GetCommand(), err) // Emit the result of execution to the audit log @@ -168,6 +236,7 @@ func (e *localExec) Start(channel ssh.Channel) (*ExecResult, error) { Code: exitCode(err), }, trace.ConvertSystemError(err) } + e.Ctx.Infof("Started local command execution: %q", e.Command) return nil, nil @@ -190,50 +259,159 @@ func (e *localExec) Wait() *ExecResult { // Emit the result of execution to the Audit Log. emitExecAuditEvent(e.Ctx, e.GetCommand(), err) - return &ExecResult{ + execResult := &ExecResult{ Command: e.GetCommand(), Code: exitCode(err), } + + return execResult +} + +// Continue will resume execution of the process after it completes its +// pre-processing routine (placed in a cgroup). +func (e *localExec) Continue() { + e.Ctx.contw.Close() + + // Set to nil so the close in the context doesn't attempt to re-close. + e.Ctx.contw = nil +} + +// PID returns the PID of the Teleport process that was re-execed. +func (e *localExec) PID() int { + return e.Cmd.Process.Pid } func (e *localExec) String() string { return fmt.Sprintf("Exec(Command=%v)", e.Command) } -// prepareInteractiveCommand configures exec.Cmd object for launching an -// interactive command (or a shell). -func prepareInteractiveCommand(ctx *ServerContext) (*exec.Cmd, error) { - var ( - err error - runShell bool - ) - // determine shell for the given OS user: - if ctx.ExecRequest.GetCommand() == "" { - runShell = true - cmdName, err := shell.GetLoginShell(ctx.Identity.Login) - ctx.ExecRequest.SetCommand(cmdName) - if err != nil { - log.Error(err) - return nil, trace.Wrap(err) - } - // in test mode short-circuit to /bin/sh - if ctx.IsTestStub { - ctx.ExecRequest.SetCommand("/bin/sh") - } +// RunCommand reads in the command to run from the parent process (over a +// pipe) then constructs and runs the command. +func RunCommand() { + // errorWriter is used to return any error message back to the client. By + // default it writes to stdout, but if a TTY is allocated, it will write + // to it. + errorWriter := os.Stdout + + // Parent sends the command payload in the third file descriptor. + cmdfd := os.NewFile(uintptr(3), "/proc/self/fd/3") + if cmdfd == nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, trace.BadParameter("command pipe not found")) } - c, err := prepareCommand(ctx) + contfd := os.NewFile(uintptr(4), "/proc/self/fd/4") + if cmdfd == nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, trace.BadParameter("continue pipe not found")) + } + + // Read in the command payload. + var b bytes.Buffer + _, err := b.ReadFrom(cmdfd) if err != nil { - return nil, trace.Wrap(err) + errorAndExit(errorWriter, teleport.RemoteCommandFailure, err) } - // this configures shell to run in 'login' mode. from openssh source: - // "If we have no command, execute the shell. In this case, the shell - // name to be passed in argv[0] is preceded by '-' to indicate that - // this is a login shell." - // https://github.com/openssh/openssh-portable/blob/master/session.c - if runShell { - c.Args = []string{"-" + filepath.Base(ctx.ExecRequest.GetCommand())} + var c execCommand + err = json.Unmarshal(b.Bytes(), &c) + if err != nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, err) } - return c, nil + + // Start constructing the the exec.Cmd. + cmd := exec.Cmd{ + Path: c.Path, + Args: c.Args, + Dir: c.Dir, + Env: c.Env, + SysProcAttr: &syscall.SysProcAttr{ + Setsid: true, + }, + } + + var tty *os.File + var pty *os.File + + // If a terminal was requested, file descriptor 4 and 5 always point to the + // PTY and TTY. Extract them and set the controlling TTY. Otherwise, connect + // std{in,out,err} directly. + if c.Terminal { + pty = os.NewFile(uintptr(5), "/proc/self/fd/5") + tty = os.NewFile(uintptr(6), "/proc/self/fd/6") + if pty == nil || tty == nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, trace.BadParameter("pty and tty not found")) + } + + cmd.Stdin = tty + cmd.Stdout = tty + cmd.Stderr = tty + + cmd.SysProcAttr.Setctty = true + cmd.SysProcAttr.Ctty = int(tty.Fd()) + + errorWriter = tty + } else { + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + } + + // Only set process credentials if requested. See comment in the + // "prepareCommand" function for more details. + if c.SetCreds { + cmd.SysProcAttr.Credential = &syscall.Credential{ + Uid: c.Uid, + Gid: c.Gid, + Groups: c.Groups, + } + } + + // Wait until the continue signal is received from Teleport signaling that + // the child process has been placed in a cgroup. + err = waitForContinue(contfd) + if err != nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, err) + } + + // If PAM is enabled, open a PAM context. + var pamContext *pam.PAM + if c.PAM.Enabled { + // Connect std{in,out,err} to the TTY if it's a shell request, otherwise + // discard std{out,err}. If this was not done, things like MOTD would be + // printed for "exec" requests. + var stdin io.Reader + var stdout io.Writer + var stderr io.Writer + if c.RequestType == sshutils.ShellRequest { + stdin = tty + stdout = tty + stderr = tty + } else { + stdin = os.Stdin + stdout = ioutil.Discard + stderr = ioutil.Discard + } + + // Open the PAM context. + pamContext, err = pam.Open(&pam.Config{ + ServiceName: c.PAM.ServiceName, + Username: c.PAM.Username, + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + }) + if err != nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, err) + } + defer pamContext.Close() + } + + // Start the command. + err = cmd.Start() + if err != nil { + errorAndExit(errorWriter, teleport.RemoteCommandFailure, err) + } + + // Wait for it to exit. + err = cmd.Wait() + errorAndExit(errorWriter, exitCode(err), err) } func (e *localExec) transformSecureCopy() error { @@ -252,7 +430,7 @@ func (e *localExec) transformSecureCopy() error { // for scp requests update the command to execute to launch teleport with // scp parameters just like openssh does. - teleportBin, err := osext.Executable() + teleportBin, err := os.Executable() if err != nil { return trace.Wrap(err) } @@ -265,17 +443,61 @@ func (e *localExec) transformSecureCopy() error { return nil } -// prepareCommand configures exec.Cmd for executing a given command within an SSH -// session. -// -// 'cmd' is the string passed as parameter to 'ssh' command, like "ls -l /" -// -// If 'cmd' does not have any spaces in it, it gets executed directly, otherwise -// it is passed to user's shell for interpretation -func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { - osUserName := ctx.Identity.Login - // configure UID & GID of the requested OS user: - osUser, err := user.Lookup(osUserName) +// prepareCommand prepares a command execution payload. +func prepareCommand(ctx *ServerContext) (*execCommand, error) { + var c execCommand + + // Get the login shell for the user (or fallback to the default). + shellPath, err := shell.GetLoginShell(ctx.Identity.Login) + if err != nil { + log.Debugf("Failed to get login shell for %v: %v. Using default: %v.", + ctx.Identity.Login, err, shell.DefaultShell) + } + if ctx.IsTestStub { + shellPath = "/bin/sh" + } + + // Save off the SSH request type. This will be used to control where to + // connect std{out,err} based on the request type: "exec" or "shell". + c.RequestType = ctx.request.Type + + // If a term was allocated before (this means it was an exec request with an + // PTY explicitly allocated) or no command was given (which means an + // interactive session was requested), then make sure "teleport exec" + // executes through a PTY. + if ctx.termAllocated || ctx.ExecRequest.GetCommand() == "" { + c.Terminal = true + } + + // If no command was given, configure a shell to run in 'login' mode. + // Otherwise, execute a command through bash. + if ctx.ExecRequest.GetCommand() == "" { + // Overwrite whatever was in the exec command (probably empty) with the shell. + ctx.ExecRequest.SetCommand(shellPath) + + // Set the path to the path of the shell. + c.Path = shellPath + + // Configure the shell to run in 'login' mode. From OpenSSH source: + // "If we have no command, execute the shell. In this case, the shell + // name to be passed in argv[0] is preceded by '-' to indicate that + // this is a login shell." + // https://github.com/openssh/openssh-portable/blob/master/session.c + c.Args = []string{"-" + filepath.Base(shellPath)} + } else { + // Execute commands like OpenSSH does: + // https://github.com/openssh/openssh-portable/blob/master/session.c + c.Path = shellPath + c.Args = []string{shellPath, "-c", ctx.ExecRequest.GetCommand()} + } + + clusterName, err := ctx.srv.GetAccessPoint().GetClusterName() + if err != nil { + return nil, trace.Wrap(err) + } + + // Lookup the UID and GID for the user. + osUser, err := user.Lookup(ctx.Identity.Login) if err != nil { return nil, trace.Wrap(err) } @@ -283,43 +505,17 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { if err != nil { return nil, trace.Wrap(err) } + c.Uid = uint32(uid) gid, err := strconv.Atoi(osUser.Gid) if err != nil { return nil, trace.Wrap(err) } + c.Gid = uint32(gid) - // get user's shell: - shell, err := shell.GetLoginShell(ctx.Identity.Login) - if err != nil { - log.Warn(err) - } - if ctx.IsTestStub { - shell = "/bin/sh" - } - - // by default, execute command using user's shell like openssh does: - // https://github.com/openssh/openssh-portable/blob/master/session.c - c := exec.Command(shell, "-c", ctx.ExecRequest.GetCommand()) - - clusterName, err := ctx.srv.GetAccessPoint().GetClusterName() - if err != nil { - return nil, trace.Wrap(err) - } - - c.Env = []string{ - "LANG=en_US.UTF-8", - getDefaultEnvPath(osUser.Uid, defaultLoginDefsPath), - "HOME=" + osUser.HomeDir, - "USER=" + osUserName, - "SHELL=" + shell, - teleport.SSHTeleportUser + "=" + ctx.Identity.TeleportUser, - teleport.SSHSessionWebproxyAddr + "=" + ctx.ProxyPublicAddress(), - teleport.SSHTeleportHostUUID + "=" + ctx.srv.ID(), - teleport.SSHTeleportClusterName + "=" + clusterName.GetClusterName(), - } + // Set the home directory for the user. c.Dir = osUser.HomeDir - // Lookup all groups the user is a member of. + // Lookup supplementary groups for the user. userGroups, err := osUser.GroupIds() if err != nil { return nil, trace.Wrap(err) @@ -336,6 +532,7 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { if len(groups) == 0 { groups = append(groups, uint32(gid)) } + c.Groups = groups // Only set process credentials if the UID/GID of the requesting user are // different than the process (Teleport). @@ -348,13 +545,8 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { // workaround this, the credentials struct is only set if the credentials // are different from the process itself. If the credentials are not, simply // pick up the ambient credentials of the process. - var credentials *syscall.Credential if strconv.Itoa(os.Getuid()) != osUser.Uid || strconv.Itoa(os.Getgid()) != osUser.Gid { - credentials = &syscall.Credential{ - Uid: uint32(uid), - Gid: uint32(gid), - Groups: groups, - } + c.SetCreds = true log.Debugf("Creating process with UID %v, GID: %v, and Groups: %v.", uid, gid, groups) } else { @@ -362,28 +554,25 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { uid, gid, groups) } - // Filling out syscall.SysProcAttr will trigger calling of certain syscalls - // during process start. - c.SysProcAttr = &syscall.SysProcAttr{ - // Call SETUID and SETGID syscalls if credentials is not nil to set the - // process UID and GID. See "man 7 credentials" for more details. - Credential: credentials, - - // Call the SETSID syscall which will "create a new session if the calling - // process is not a process group leader". See "man 2 setsid" for more details. - Setsid: true, + // Create environment for user. + c.Env = []string{ + "LANG=en_US.UTF-8", + getDefaultEnvPath(osUser.Uid, defaultLoginDefsPath), + "HOME=" + osUser.HomeDir, + "USER=" + ctx.Identity.Login, + "SHELL=" + shellPath, + teleport.SSHTeleportUser + "=" + ctx.Identity.TeleportUser, + teleport.SSHSessionWebproxyAddr + "=" + ctx.ProxyPublicAddress(), + teleport.SSHTeleportHostUUID + "=" + ctx.srv.ID(), + teleport.SSHTeleportClusterName + "=" + clusterName.GetClusterName(), } - // apply environment variables passed from the client + // Apply environment variables passed in from client. for n, v := range ctx.env { c.Env = append(c.Env, fmt.Sprintf("%s=%s", n, v)) } - // if a terminal was allocated, apply terminal type variable - if ctx.session != nil { - c.Env = append(c.Env, fmt.Sprintf("TERM=%v", ctx.session.term.GetTermType())) - } - // apply SSH_xx environment variables + // Apply SSH_* environment variables. remoteHost, remotePort, err := net.SplitHostPort(ctx.Conn.RemoteAddr().String()) if err != nil { log.Warn(err) @@ -406,8 +595,28 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { } } - // if the server allows reading in of ~/.tsh/environment read it in - // and pass environment variables along to new session + // If a terminal was allocated, set terminal type variable. + if ctx.session != nil { + c.Env = append(c.Env, fmt.Sprintf("TERM=%v", ctx.session.term.GetTermType())) + } + + // If the command is being prepared for local execution, check if PAM should + // be called. + if ctx.srv.Component() == teleport.ComponentNode { + conf, err := ctx.srv.GetPAM() + if err != nil { + return nil, trace.Wrap(err) + } + + c.PAM = &pamCommand{ + Enabled: conf.Enabled, + ServiceName: conf.ServiceName, + Username: ctx.Identity.Login, + } + } + + // If the server allows reading in of ~/.tsh/environment read it in + // and pass environment variables along to new session. if ctx.srv.PermitUserEnvironment() { filename := filepath.Join(osUser.HomeDir, ".tsh", "environment") userEnvs, err := utils.ReadEnvironmentFile(filename) @@ -416,7 +625,85 @@ func prepareCommand(ctx *ServerContext) (*exec.Cmd, error) { } c.Env = append(c.Env, userEnvs...) } - return c, nil + + return &c, nil +} + +// configureCommand creates a command fully configured to execute. +func configureCommand(ctx *ServerContext) (*exec.Cmd, error) { + var err error + + // Create and marshal command to execute. + cmdmsg, err := prepareCommand(ctx) + if err != nil { + return nil, trace.Wrap(err) + } + cmdbytes, err := json.Marshal(cmdmsg) + if err != nil { + return nil, trace.Wrap(err) + } + + // Write command bytes to pipe. The child process will read the command + // to execute from this pipe. + _, err = io.Copy(ctx.cmdw, bytes.NewReader(cmdbytes)) + if err != nil { + return nil, trace.Wrap(err) + } + err = ctx.cmdw.Close() + if err != nil { + return nil, trace.Wrap(err) + } + // Set to nil so the close in the context doesn't attempt to re-close. + ctx.cmdw = nil + + // Re-execute Teleport and pass along the allocated PTY as well as the + // command reader from where Teleport will know how to re-spawn itself. + executable, err := os.Executable() + if err != nil { + return nil, trace.Wrap(err) + } + + // Build the list of arguments to have Teleport re-exec itself. The "-d" flag + // is appended if Teleport is running in debug mode. + args := []string{executable, teleport.ExecSubCommand} + + // Build the "teleport exec" command. + return &exec.Cmd{ + Path: executable, + Args: args, + Dir: cmdmsg.Dir, + ExtraFiles: []*os.File{ + ctx.cmdr, + ctx.contr, + }, + }, nil +} + +// waitForContinue will wait 10 seconds for the continue signal, if not +// received, it will stop waiting and exit. +func waitForContinue(contfd *os.File) error { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + // Reading from the continue file descriptor will block until it's closed. It + // won't be closed until the parent has placed it in a cgroup. + var r bytes.Buffer + r.ReadFrom(contfd) + + // Continue signal has been processed, signal to continue execution. + cancel() + }() + + // Wait for 10 seconds and then timeout if no continue signal has been sent. + timeout := time.NewTimer(10 * time.Second) + defer timeout.Stop() + + select { + case <-timeout.C: + return trace.BadParameter("timed out waiting for continue signal") + case <-ctx.Done(): + } + return nil } // remoteExec is used to run an "exec" SSH request and return the result. @@ -480,6 +767,16 @@ func (r *remoteExec) Wait() *ExecResult { } } +// Continue does nothing for remote command execution. +func (r *remoteExec) Continue() { + return +} + +// PID returns an invalid PID for remotExec. +func (r *remoteExec) PID() int { + return 0 +} + func emitExecAuditEvent(ctx *ServerContext, cmd string, execErr error) { // Report the result of this exec event to the audit logger. auditLog := ctx.srv.GetAuditLog() @@ -658,3 +955,13 @@ func exitCode(err error) int { return teleport.RemoteCommandFailure } } + +// errorAndExit writes the error to the io.Writer (stdout or a TTY) and +// exits with the given code. +func errorAndExit(w io.Writer, code int, err error) { + s := fmt.Sprintf("Failed to launch shell: %v.\r\n", err) + if err != nil { + io.Copy(w, bytes.NewBufferString(s)) + } + os.Exit(code) +} diff --git a/lib/srv/exec_test.go b/lib/srv/exec_test.go index 2927c3ad216..4aacd0196e5 100644 --- a/lib/srv/exec_test.go +++ b/lib/srv/exec_test.go @@ -22,6 +22,7 @@ import ( "io" "net" "os" + os_exec "os/exec" "os/user" "path" "path/filepath" @@ -35,10 +36,12 @@ import ( "github.com/gravitational/teleport/lib/auth" authority "github.com/gravitational/teleport/lib/auth/testauthority" "github.com/gravitational/teleport/lib/backend/lite" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/pam" "github.com/gravitational/teleport/lib/services" rsession "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/sshutils" "github.com/gravitational/teleport/lib/utils" "github.com/docker/docker/pkg/term" @@ -53,11 +56,27 @@ type ExecSuite struct { ctx *ServerContext localAddr net.Addr remoteAddr net.Addr + a *auth.AuthServer } var _ = check.Suite(&ExecSuite{}) var _ = fmt.Printf +// TestMain will re-execute Teleport to run a command if "exec" is passed to +// it as an argument. Otherwise it will run tests as normal. +func TestMain(m *testing.M) { + // If the test is re-executing itself, execute the command that comes over + // the pipe. + if len(os.Args) == 2 && os.Args[1] == teleport.ExecSubCommand { + RunCommand() + return + } + + // Otherwise run tests as normal. + code := m.Run() + os.Exit(code) +} + func (s *ExecSuite) SetUpSuite(c *check.C) { utils.InitLoggerForTests(testing.Verbose()) @@ -70,20 +89,20 @@ func (s *ExecSuite) SetUpSuite(c *check.C) { c.Assert(err, check.IsNil) c.Assert(err, check.IsNil) - a, err := auth.NewAuthServer(&auth.InitConfig{ + s.a, err = auth.NewAuthServer(&auth.InitConfig{ Backend: bk, Authority: authority.New(), ClusterName: clusterName, }) c.Assert(err, check.IsNil) - a.SetClusterName(clusterName) + s.a.SetClusterName(clusterName) // set static tokens staticTokens, err := services.NewStaticTokens(services.StaticTokensSpecV2{ StaticTokens: []services.ProvisionTokenV1{}, }) c.Assert(err, check.IsNil) - err = a.SetStaticTokens(staticTokens) + err = s.a.SetStaticTokens(staticTokens) c.Assert(err, check.IsNil) dir := c.MkDir() @@ -94,7 +113,7 @@ func (s *ExecSuite) SetUpSuite(c *check.C) { s.ctx = &ServerContext{ IsTestStub: true, srv: &fakeServer{ - accessPoint: a, + accessPoint: s.a, auditLog: &fakeLog{}, id: "00000000-0000-0000-0000-000000000000", }, @@ -104,6 +123,9 @@ func (s *ExecSuite) SetUpSuite(c *check.C) { s.ctx.Identity.TeleportUser = "galt" s.ctx.Conn = &ssh.ServerConn{Conn: s} s.ctx.ExecRequest = &localExec{Ctx: s.ctx} + s.ctx.request = &ssh.Request{ + Type: sshutils.ExecRequest, + } term, err := newLocalTerminal(s.ctx) c.Assert(err, check.IsNil) @@ -129,15 +151,15 @@ func (s *ExecSuite) TestOSCommandPrep(c *check.C) { "SSH_SESSION_WEBPROXY_ADDR=:3080", "SSH_TELEPORT_HOST_UUID=00000000-0000-0000-0000-000000000000", "SSH_TELEPORT_CLUSTER_NAME=localhost", - "TERM=xterm", "SSH_CLIENT=10.0.0.5 4817 3022", "SSH_CONNECTION=10.0.0.5 4817 127.0.0.1 3022", fmt.Sprintf("SSH_TTY=%v", s.ctx.session.term.TTY().Name()), "SSH_SESSION_ID=xxx", + "TERM=xterm", } - // empty command (simple shell) - cmd, err := prepareInteractiveCommand(s.ctx) + // Empty command (simple shell). + cmd, err := prepareCommand(s.ctx) c.Assert(err, check.IsNil) c.Assert(cmd, check.NotNil) c.Assert(cmd.Path, check.Equals, "/bin/sh") @@ -145,7 +167,7 @@ func (s *ExecSuite) TestOSCommandPrep(c *check.C) { c.Assert(cmd.Dir, check.Equals, s.usr.HomeDir) c.Assert(cmd.Env, check.DeepEquals, expectedEnv) - // non-empty command (exec a prog) + // Non-empty command (exec a prog). s.ctx.IsTestStub = true s.ctx.ExecRequest.SetCommand("ls -lh /etc") cmd, err = prepareCommand(s.ctx) @@ -156,7 +178,7 @@ func (s *ExecSuite) TestOSCommandPrep(c *check.C) { c.Assert(cmd.Dir, check.Equals, s.usr.HomeDir) c.Assert(cmd.Env, check.DeepEquals, expectedEnv) - // command without args + // Command without args. s.ctx.ExecRequest.SetCommand("top") cmd, err = prepareCommand(s.ctx) c.Assert(err, check.IsNil) @@ -214,7 +236,76 @@ func (s *ExecSuite) TestEmitExecAuditEvent(c *check.C) { } } -// implementation of ssh.Conn interface +// TestContinue tests if the process hangs if a continue signal is not sent +// and makes sure the process continues once it has been sent. +func (s *ExecSuite) TestContinue(c *check.C) { + var err error + + lsPath, err := os_exec.LookPath("ls") + c.Assert(err, check.IsNil) + + // Create a fake context that will be used to configure a command that will + // re-exec "ls". + ctx := &ServerContext{ + IsTestStub: true, + srv: &fakeServer{ + accessPoint: s.a, + auditLog: &fakeLog{}, + id: "00000000-0000-0000-0000-000000000000", + }, + } + ctx.Identity.Login = s.usr.Username + ctx.Identity.TeleportUser = "galt" + ctx.Conn = &ssh.ServerConn{Conn: s} + ctx.ExecRequest = &localExec{ + Ctx: ctx, + Command: lsPath, + } + ctx.cmdr, ctx.cmdw, err = os.Pipe() + c.Assert(err, check.IsNil) + ctx.contr, ctx.contw, err = os.Pipe() + c.Assert(err, check.IsNil) + ctx.request = &ssh.Request{ + Type: sshutils.ExecRequest, + } + + // Create an exec.Cmd to execute through Teleport. + cmd, err := configureCommand(ctx) + c.Assert(err, check.IsNil) + + // Create a context that will be used to signal that execution is complete. + doneContext, doneCancel := context.WithCancel(context.Background()) + + // Re-execute Teleport and run "ls". Signal over the context when execution + // is complete. + go func() { + cmd.Run() + doneCancel() + }() + + // Wait for the process. Since the continue pipe has not been closed, the + // process should not have exited yet. + select { + case <-doneContext.Done(): + c.Fatalf("Process exited before continue.") + case <-time.After(5 * time.Second): + } + + // Close the continue pipe to signal to Teleport to now execute the + // requested program. + err = ctx.contw.Close() + c.Assert(err, check.IsNil) + + // Program should have executed now. If the complete signal has not come + // over the context, something failed. + select { + case <-time.After(5 * time.Second): + c.Fatalf("Timed out waiting for process to finish.") + case <-doneContext.Done(): + } +} + +// Implementation of ssh.Conn interface. func (s *ExecSuite) User() string { return s.usr.Username } func (s *ExecSuite) SessionID() []byte { return []byte{1, 2, 3} } func (s *ExecSuite) ClientVersion() []byte { return []byte{1} } @@ -259,6 +350,12 @@ func (f *fakeTerminal) Wait() (*ExecResult, error) { return nil, nil } +// Continue will resume execution of the process after it completes its +// pre-processing routine (placed in a cgroup). +func (f *fakeTerminal) Continue() { + return +} + // Kill will force kill the terminal. func (f *fakeTerminal) Kill() error { return nil @@ -274,6 +371,11 @@ func (f *fakeTerminal) TTY() *os.File { return f.f } +// PID returns the PID of the Teleport process that was re-execed. +func (f *fakeTerminal) PID() int { + return 1 +} + // Close will free resources associated with the terminal. func (f *fakeTerminal) Close() error { return f.f.Close() @@ -375,6 +477,10 @@ func (f *fakeServer) UseTunnel() bool { return false } +func (f *fakeServer) GetBPF() bpf.BPF { + return &bpf.NOP{} +} + // fakeLog is used in tests to obtain the last event emit to the Audit Log. type fakeLog struct { lastEvent events.EventFields diff --git a/lib/srv/forward/sshserver.go b/lib/srv/forward/sshserver.go index b4668b95b96..0d357d0c251 100644 --- a/lib/srv/forward/sshserver.go +++ b/lib/srv/forward/sshserver.go @@ -28,6 +28,7 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/pam" @@ -366,6 +367,13 @@ func (s *Server) UseTunnel() bool { return s.useTunnel } +// GetBPF returns the BPF service used by enhanced session recording. BPF +// for the forwarding server makes no sense (it has to run on the actual +// node), so return a NOP implementation. +func (s Server) GetBPF() bpf.BPF { + return &bpf.NOP{} +} + // GetInfo returns a services.Server that represents this server. func (s *Server) GetInfo() services.Server { return &services.ServerV2{ diff --git a/lib/srv/regular/sshserver.go b/lib/srv/regular/sshserver.go index 547ceb2b399..19d40d70826 100644 --- a/lib/srv/regular/sshserver.go +++ b/lib/srv/regular/sshserver.go @@ -38,6 +38,7 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" "github.com/gravitational/teleport/lib/limiter" @@ -148,6 +149,9 @@ type Server struct { // fips means Teleport started in a FedRAMP/FIPS 140-2 compliant // configuration. fips bool + + // ebpf is the service used for enhanced session recording. + ebpf bpf.BPF } // GetClock returns server clock implementation @@ -193,6 +197,11 @@ func (s *Server) UseTunnel() bool { return s.useTunnel } +// GetBPF returns the BPF service used by enhanced session recording. +func (s *Server) GetBPF() bpf.BPF { + return s.ebpf +} + // isAuditedAtProxy returns true if sessions are being recorded at the proxy // and this is a Teleport node. func (s *Server) isAuditedAtProxy() bool { @@ -419,6 +428,13 @@ func SetFIPS(fips bool) ServerOption { } } +func SetBPF(ebpf bpf.BPF) ServerOption { + return func(s *Server) error { + s.ebpf = ebpf + return nil + } +} + // New returns an unstarted server func New(addr utils.NetAddr, hostname string, diff --git a/lib/srv/regular/sshserver_test.go b/lib/srv/regular/sshserver_test.go index 7638b023d1c..ede443baa8e 100644 --- a/lib/srv/regular/sshserver_test.go +++ b/lib/srv/regular/sshserver_test.go @@ -36,12 +36,14 @@ import ( "github.com/gravitational/teleport" "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/limiter" "github.com/gravitational/teleport/lib/pam" "github.com/gravitational/teleport/lib/reversetunnel" "github.com/gravitational/teleport/lib/services" sess "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/srv" "github.com/gravitational/teleport/lib/sshutils" "github.com/gravitational/teleport/lib/utils" @@ -81,6 +83,18 @@ var wildcardAllow = services.Labels{ var _ = fmt.Printf var _ = Suite(&SrvSuite{}) +// TestMain will re-execute Teleport to run a command if "exec" is passed to +// it as an argument. Otherwise it will run tests as normal. +func TestMain(m *testing.M) { + if len(os.Args) == 2 && os.Args[1] == teleport.ExecSubCommand { + srv.RunCommand() + return + } + + code := m.Run() + os.Exit(code) +} + func (s *SrvSuite) SetUpSuite(c *C) { var err error @@ -158,6 +172,7 @@ func (s *SrvSuite) SetUpTest(c *C) { Command: []string{"expr", "1", "+", "3"}}, }, ), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) s.srv = srv @@ -586,6 +601,7 @@ func (s *SrvSuite) TestProxyReverseTunnel(c *C) { SetAuditLog(s.nodeClient), SetNamespace(defaults.Namespace), SetPAMConfig(&pam.Config{Enabled: false}), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) c.Assert(proxy.Start(), IsNil) @@ -659,6 +675,7 @@ func (s *SrvSuite) TestProxyReverseTunnel(c *C) { SetNamespace(defaults.Namespace), SetPAMConfig(&pam.Config{Enabled: false}), SetUUID(bobAddr), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) c.Assert(err, IsNil) @@ -745,6 +762,7 @@ func (s *SrvSuite) TestProxyRoundRobin(c *C) { SetAuditLog(s.nodeClient), SetNamespace(defaults.Namespace), SetPAMConfig(&pam.Config{Enabled: false}), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) c.Assert(proxy.Start(), IsNil) @@ -846,6 +864,7 @@ func (s *SrvSuite) TestProxyDirectAccess(c *C) { SetAuditLog(s.nodeClient), SetNamespace(defaults.Namespace), SetPAMConfig(&pam.Config{Enabled: false}), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) c.Assert(proxy.Start(), IsNil) @@ -956,6 +975,7 @@ func (s *SrvSuite) TestLimiter(c *C) { SetAuditLog(s.nodeClient), SetNamespace(defaults.Namespace), SetPAMConfig(&pam.Config{Enabled: false}), + SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) c.Assert(srv.Start(), IsNil) diff --git a/lib/srv/sess.go b/lib/srv/sess.go index a853b2f4b3d..b3542d0d7b7 100644 --- a/lib/srv/sess.go +++ b/lib/srv/sess.go @@ -27,9 +27,9 @@ import ( "golang.org/x/crypto/ssh" "github.com/gravitational/teleport" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" - "github.com/gravitational/teleport/lib/pam" "github.com/gravitational/teleport/lib/services" rsession "github.com/gravitational/teleport/lib/session" "github.com/gravitational/teleport/lib/sshutils" @@ -73,7 +73,9 @@ type SessionRegistry struct { // log holds the structured logger log *logrus.Entry - // sessions holds a map between session ID and the session object. + // sessions holds a map between session ID and the session object. Used to + // find active sessions as well as close all sessions when the registry + // is closing. sessions map[rsession.ID]*session // srv refers to the upon which this session registry is created. @@ -84,6 +86,7 @@ func NewSessionRegistry(srv Server) (*SessionRegistry, error) { if srv.GetSessionServer() == nil { return nil, trace.BadParameter("session server is required") } + return &SessionRegistry{ log: logrus.WithFields(logrus.Fields{ trace.Component: teleport.Component(teleport.ComponentSession, srv.Component()), @@ -99,6 +102,17 @@ func (s *SessionRegistry) addSession(sess *session) { s.sessions[sess.id] = sess } +func (s *SessionRegistry) removeSession(sess *session) { + s.Lock() + defer s.Unlock() + delete(s.sessions, sess.id) +} + +func (s *SessionRegistry) findSession(id rsession.ID) (*session, bool) { + sess, found := s.sessions[id] + return sess, found +} + func (s *SessionRegistry) Close() { s.Lock() defer s.Unlock() @@ -178,15 +192,42 @@ func (s *SessionRegistry) OpenSession(ch ssh.Channel, req *ssh.Request, ctx *Ser } ctx.session = sess s.addSession(sess) - ctx.Infof("Creating session %v.", sid) + ctx.Infof("Creating (interactive) session %v.", sid) - if err := sess.start(ch, ctx); err != nil { + // Start an interactive session (TTY attached). Close the session if an error + // occurs, otherwise it will be closed by the callee. + if err := sess.startInteractive(ch, ctx); err != nil { sess.Close() return trace.Wrap(err) } return nil } +// OpenExecSession opens an non-interactive exec session. +func (s *SessionRegistry) OpenExecSession(channel ssh.Channel, req *ssh.Request, ctx *ServerContext) error { + // Create a new session ID. These sessions can not be joined so no point in + // looking for an exisiting one. + sessionID := rsession.NewID() + + // This logic allows concurrent request to create a new session + // to fail, what is ok because we should never have this condition. + sess, err := newSession(sessionID, s, ctx) + if err != nil { + return trace.Wrap(err) + } + ctx.Infof("Creating (exec) session %v.", sessionID) + + // Start a non-interactive session (TTY attached). Close the session if an error + // occurs, otherwise it will be closed by the callee. + err = sess.startExec(channel, ctx) + defer sess.Close() + if err != nil { + return trace.Wrap(err) + } + + return nil +} + // emitSessionLeaveEvent emits a session leave event to both the Audit Log as // well as sending a "x-teleport-event" global request on the SSH connection. func (s *SessionRegistry) emitSessionLeaveEvent(party *party) { @@ -250,17 +291,19 @@ func (s *SessionRegistry) leaveSession(party *party) error { s.log.Infof("Session %v will be garbage collected.", sess.id) // no more people left? Need to end the session! - s.Lock() - delete(s.sessions, sess.id) - s.Unlock() + s.removeSession(sess) - // send an event indicating that this session has ended - sess.recorder.GetAuditLog().EmitAuditEvent(events.SessionEnd, events.EventFields{ - events.SessionEventID: string(sess.id), - events.SessionServerID: party.ctx.srv.HostUUID(), - events.EventUser: party.user, - events.EventNamespace: s.srv.GetNamespace(), - }) + // Emit a session.end event for this (interactive) session. + eventFields := events.EventFields{ + events.SessionEventID: string(sess.id), + events.SessionServerID: party.ctx.srv.HostUUID(), + events.EventUser: party.user, + events.EventNamespace: s.srv.GetNamespace(), + events.SessionInteractive: true, + events.SessionEnhancedRecording: sess.hasEnhancedRecording, + events.SessionParticipants: sess.exportParticipants(), + } + sess.recorder.GetAuditLog().EmitAuditEvent(events.SessionEnd, eventFields) // close recorder to free up associated resources // and flush data @@ -380,11 +423,6 @@ func (s *SessionRegistry) broadcastResult(sid rsession.ID, r ExecResult) error { return nil } -func (s *SessionRegistry) findSession(id rsession.ID) (*session, bool) { - sess, found := s.sessions[id] - return sess, found -} - // session struct describes an active (in progress) SSH session. These sessions // are managed by 'SessionRegistry' containers which are attached to SSH servers. type session struct { @@ -402,9 +440,15 @@ type session struct { // this writer is used to broadcast terminal I/O to different clients writer *multiWriter - // parties are connected lients/users + // parties is the set of current connected clients/users. This map may grow + // and shrink as members join and leave the session. parties map[rsession.ID]*party + // participants is the set of users that have joined this session. Users are + // never removed from this map as it's used to report the full list of + // participants at the end of a session. + participants map[rsession.ID]*party + term Terminal // closeC channel is used to kill all goroutines owned @@ -427,6 +471,10 @@ type session struct { closeOnce sync.Once recorder events.SessionRecorder + + // hasEnhancedRecording returns true if this session has enhanced session + // recording events associated. + hasEnhancedRecording bool } // newSession creates a new session with a given ID within a given context. @@ -484,24 +532,32 @@ func newSession(id rsession.ID, r *SessionRegistry, ctx *ServerContext) (*sessio log: logrus.WithFields(logrus.Fields{ trace.Component: teleport.Component(teleport.ComponentSession, r.srv.Component()), }), - id: id, - registry: r, - parties: make(map[rsession.ID]*party), - writer: newMultiWriter(), - login: ctx.Identity.Login, - closeC: make(chan bool), - lingerTTL: defaults.SessionIdlePeriod, + id: id, + registry: r, + parties: make(map[rsession.ID]*party), + participants: make(map[rsession.ID]*party), + writer: newMultiWriter(), + login: ctx.Identity.Login, + closeC: make(chan bool), + lingerTTL: defaults.SessionIdlePeriod, } return sess, nil } -// isLingering returns true if every party has left this session. Occurs -// under a lock. -func (s *session) isLingering() bool { - s.Lock() - defer s.Unlock() +// ID returns a string representation of the session ID. +func (s *session) ID() string { + return s.id.String() +} - return len(s.parties) == 0 +// PID returns the PID of the Teleport process under which the shell is running. +func (s *session) PID() int { + return s.term.PID() +} + +// Recorder returns a events.SessionRecorder which can be used to emit events +// to a session as well as the audit log. +func (s *session) Recorder() events.SessionRecorder { + return s.recorder } // Close ends the active session forcing all clients to disconnect and freeing all resources @@ -533,19 +589,24 @@ func (s *session) Close() error { return nil } -func isDiscardAuditLog(alog events.IAuditLog) bool { - _, ok := alog.(*events.DiscardAuditLog) - return ok +// isLingering returns true if every party has left this session. Occurs +// under a lock. +func (s *session) isLingering() bool { + s.Lock() + defer s.Unlock() + + return len(s.parties) == 0 } -// start starts a new interactive process (or a shell) in the current session -func (s *session) start(ch ssh.Channel, ctx *ServerContext) error { +// startInteractive starts a new interactive process (or a shell) in the +// current session. +func (s *session) startInteractive(ch ssh.Channel, ctx *ServerContext) error { var err error // create a new "party" (connected client) p := newParty(s, ch, ctx) - // get the audit log from the server and create a session recorder. this will + // Get the audit log from the server and create a session recorder. this will // be a discard audit log if the proxy is in recording mode and a teleport // node so we don't create double recordings. auditLog := s.registry.srv.GetAuditLog() @@ -566,30 +627,6 @@ func (s *session) start(ch ssh.Channel, ctx *ServerContext) error { } s.writer.addWriter("session-recorder", s.recorder, true) - // If this code is running on a Teleport node and PAM is enabled, then open a - // PAM context. - var pamContext *pam.PAM - if ctx.srv.Component() == teleport.ComponentNode { - conf, err := s.registry.srv.GetPAM() - if err != nil { - return trace.Wrap(err) - } - - if conf.Enabled == true { - pamContext, err = pam.Open(&pam.Config{ - ServiceName: conf.ServiceName, - Username: ctx.Identity.Login, - Stdin: ch, - Stderr: s.writer, - Stdout: s.writer, - }) - if err != nil { - return trace.Wrap(err) - } - ctx.Debugf("Opening PAM context for session %v.", s.id) - } - } - // allocate a terminal or take the one previously allocated via a // seaprate "allocate TTY" SSH request if ctx.GetTerm() != nil { @@ -610,17 +647,45 @@ func (s *session) start(ch ssh.Channel, ctx *ServerContext) error { return trace.Wrap(err) } + // Open a BPF recording session. If BPF was not configured, not available, + // or running in a recording proxy, OpenSession is a NOP. + sessionContext := &bpf.SessionContext{ + PID: s.term.PID(), + AuditLog: s.recorder.GetAuditLog(), + Namespace: ctx.srv.GetNamespace(), + SessionID: s.id.String(), + ServerID: ctx.srv.HostUUID(), + Login: ctx.Identity.Login, + User: ctx.Identity.TeleportUser, + Events: ctx.Identity.RoleSet.EnhancedRecordingSet(), + } + cgroupID, err := ctx.srv.GetBPF().OpenSession(sessionContext) + if err != nil { + ctx.Errorf("Failed to open enhanced recording (interactive) session: %v: %v.", s.id, err) + return trace.Wrap(err) + } + + // If a cgroup ID was assigned then enhanced session recording was enabled. + if cgroupID > 0 { + s.hasEnhancedRecording = true + } + + // Process has been placed in a cgroup, continue execution. + s.term.Continue() + params := s.term.GetTerminalParams() - // Emit "new session created" event. + // Emit "new session created" event for the interactive session. eventFields := events.EventFields{ - events.EventNamespace: ctx.srv.GetNamespace(), - events.SessionEventID: string(s.id), - events.SessionServerID: ctx.srv.HostUUID(), - events.EventLogin: ctx.Identity.Login, - events.EventUser: ctx.Identity.TeleportUser, - events.RemoteAddr: ctx.Conn.RemoteAddr().String(), - events.TerminalSize: params.Serialize(), + events.EventNamespace: ctx.srv.GetNamespace(), + events.SessionEventID: string(s.id), + events.SessionServerID: ctx.srv.HostUUID(), + events.EventLogin: ctx.Identity.Login, + events.EventUser: ctx.Identity.TeleportUser, + events.RemoteAddr: ctx.Conn.RemoteAddr().String(), + events.TerminalSize: params.Serialize(), + events.SessionServerHostname: ctx.srv.GetInfo().GetHostname(), + events.SessionServerLabels: ctx.srv.GetInfo().GetAllLabels(), } // Local address only makes sense for non-tunnel nodes. if !ctx.srv.UseTunnel() { @@ -668,22 +733,11 @@ func (s *session) start(ch ssh.Channel, ctx *ServerContext) error { case <-doneCh: } - // If this code is running on a Teleport node and PAM is enabled, close the context. - if ctx.srv.Component() == teleport.ComponentNode { - conf, err := s.registry.srv.GetPAM() - if err != nil { - ctx.Errorf("Unable to get PAM configuration from server: %v", err) - return - } - - if conf.Enabled == true { - err = pamContext.Close() - if err != nil { - ctx.Errorf("Unable to close PAM context for session: %v: %v", s.id, err) - return - } - ctx.Debugf("Closing PAM context for session: %v.", s.id) - } + // Close the BPF recording session. If BPF was not configured, not available, + // or running in a recording proxy, this is simply a NOP. + err = ctx.srv.GetBPF().CloseSession(sessionContext) + if err != nil { + ctx.Errorf("Failed to close enhanced recording (interactive) session: %v: %v.", s.id, err) } if result != nil { @@ -706,6 +760,141 @@ func (s *session) start(ch ssh.Channel, ctx *ServerContext) error { return nil } +func (s *session) startExec(channel ssh.Channel, ctx *ServerContext) error { + var err error + + // Get the audit log from the server and create a session recorder. this will + // be a discard audit log if the proxy is in recording mode and a teleport + // node so we don't create double recordings. + auditLog := s.registry.srv.GetAuditLog() + if auditLog == nil || isDiscardAuditLog(auditLog) { + s.recorder = &events.DiscardRecorder{} + } else { + s.recorder, err = events.NewForwardRecorder(events.ForwardRecorderConfig{ + DataDir: filepath.Join(ctx.srv.GetDataDir(), teleport.LogsDir), + SessionID: s.id, + Namespace: ctx.srv.GetNamespace(), + RecordSessions: ctx.ClusterConfig.GetSessionRecording() != services.RecordOff, + Component: teleport.Component(teleport.ComponentSession, ctx.srv.Component()), + ForwardTo: auditLog, + }) + if err != nil { + return trace.Wrap(err) + } + } + + // Emit a session.start event for the exec session. + eventFields := events.EventFields{ + events.EventNamespace: ctx.srv.GetNamespace(), + events.SessionEventID: string(s.id), + events.SessionServerID: ctx.srv.HostUUID(), + events.EventLogin: ctx.Identity.Login, + events.EventUser: ctx.Identity.TeleportUser, + events.RemoteAddr: ctx.Conn.RemoteAddr().String(), + events.SessionServerHostname: ctx.srv.GetInfo().GetHostname(), + events.SessionServerLabels: ctx.srv.GetInfo().GetAllLabels(), + } + // Local address only makes sense for non-tunnel nodes. + if !ctx.srv.UseTunnel() { + eventFields[events.LocalAddr] = ctx.Conn.LocalAddr().String() + } + s.recorder.GetAuditLog().EmitAuditEvent(events.SessionStart, eventFields) + + // Start execution. If the program failed to start, send that result back. + // Note this is a partial start. Teleport will have re-exec'ed itself and + // wait until it's been placed in a cgroup and told to continue. + result, err := ctx.ExecRequest.Start(channel) + if err != nil { + return trace.Wrap(err) + } + if result != nil { + ctx.Debugf("Exec request (%v) result: %v.", ctx.ExecRequest, result) + ctx.SendExecResult(*result) + } + + // Open a BPF recording session. If BPF was not configured, not available, + // or running in a recording proxy, OpenSession is a NOP. + sessionContext := &bpf.SessionContext{ + PID: ctx.ExecRequest.PID(), + AuditLog: s.recorder.GetAuditLog(), + Namespace: ctx.srv.GetNamespace(), + SessionID: string(s.id), + ServerID: ctx.srv.HostUUID(), + Login: ctx.Identity.Login, + User: ctx.Identity.TeleportUser, + Events: ctx.Identity.RoleSet.EnhancedRecordingSet(), + } + cgroupID, err := ctx.srv.GetBPF().OpenSession(sessionContext) + if err != nil { + ctx.Errorf("Failed to open enhanced recording (exec) session: %v: %v.", ctx.ExecRequest.GetCommand(), err) + return trace.Wrap(err) + } + + // If a cgroup ID was assigned then enhanced session recording was enabled. + if cgroupID > 0 { + s.hasEnhancedRecording = true + } + + // Process has been placed in a cgroup, continue execution. + ctx.ExecRequest.Continue() + + // Process is running, wait for it to stop. + go func() { + result = ctx.ExecRequest.Wait() + if result != nil { + ctx.SendExecResult(*result) + } + + // Wait a little bit to let all events filter through before closing the + // BPF session so everything can be recorded. + time.Sleep(2 * time.Second) + + // Close the BPF recording session. If BPF was not configured, not available, + // or running in a recording proxy, this is simply a NOP. + err = ctx.srv.GetBPF().CloseSession(sessionContext) + if err != nil { + ctx.Errorf("Failed to close enhanced recording (exec) session: %v: %v.", s.id, err) + } + + // Remove the session from the in-memory map. + s.registry.removeSession(s) + + // Emit a session.end event for this (exec) session. + eventFields := events.EventFields{ + events.SessionEventID: string(s.id), + events.SessionServerID: ctx.srv.HostUUID(), + events.EventNamespace: ctx.srv.GetNamespace(), + events.SessionInteractive: false, + events.SessionEnhancedRecording: s.hasEnhancedRecording, + events.SessionParticipants: []string{ + ctx.Identity.TeleportUser, + }, + } + s.recorder.GetAuditLog().EmitAuditEvent(events.SessionEnd, eventFields) + + // Close recorder to free up associated resources and flush data. + s.recorder.Close() + + // Close the session. + err = s.Close() + if err != nil { + ctx.Errorf("Failed to close session %v: %v.", s.id, err) + } + + // Remove the session from the backend. + if ctx.srv.GetSessionServer() != nil { + err := ctx.srv.GetSessionServer().DeleteSession(ctx.srv.GetNamespace(), s.id) + if err != nil { + ctx.Errorf("Failed to remove active session: %v: %v. "+ + "Access to backend may be degraded, check connectivity to backend.", + s.id, err) + } + } + }() + + return nil +} + func (s *session) broadcastResult(r ExecResult) { for _, p := range s.parties { p.ctx.SendExecResult(r) @@ -774,6 +963,19 @@ func (s *session) exportPartyMembers() []rsession.Party { return partyList } +// exportParticipants returns a list of all members that joined the party. +func (s *session) exportParticipants() []string { + s.Lock() + defer s.Unlock() + + var participants []string + for _, p := range s.participants { + participants = append(participants, p.user) + } + + return participants +} + // heartbeat will loop as long as the session is not closed and mark it as // active and update the list of party members. If the session are recorded at // the proxy, then this function does nothing as it's counterpart @@ -826,6 +1028,7 @@ func (s *session) addPartyMember(p *party) { defer s.Unlock() s.parties[p.id] = p + s.participants[p.id] = p } // addParty is called when a new party joins the session. @@ -1027,3 +1230,8 @@ func (p *party) Close() (err error) { }) return err } + +func isDiscardAuditLog(alog events.IAuditLog) bool { + _, ok := alog.(*events.DiscardAuditLog) + return ok +} diff --git a/lib/srv/term.go b/lib/srv/term.go index 912ca376378..689e8084426 100644 --- a/lib/srv/term.go +++ b/lib/srv/term.go @@ -58,6 +58,10 @@ type Terminal interface { // Wait will block until the terminal is complete. Wait() (*ExecResult, error) + // Continue will resume execution of the process after it completes its + // pre-processing routine (placed in a cgroup). + Continue() + // Kill will force kill the terminal. Kill() error @@ -67,6 +71,9 @@ type Terminal interface { // TTY returns the TTY backing the terminal. TTY() *os.File + // PID returns the PID of the Teleport process that was re-execed. + PID() int + // Close will free resources associated with the terminal. Close() error @@ -120,6 +127,8 @@ type terminal struct { pty *os.File tty *os.File + pid int + termType string params rsession.TerminalParams } @@ -160,25 +169,28 @@ func (t *terminal) AddParty(delta int) { // Run will run the terminal. func (t *terminal) Run() error { + var err error defer t.closeTTY() - cmd, err := prepareInteractiveCommand(t.ctx) + // Create the command that will actually execute. + t.cmd, err = configureCommand(t.ctx) if err != nil { return trace.Wrap(err) } - t.cmd = cmd - cmd.Stdout = t.tty - cmd.Stdin = t.tty - cmd.Stderr = t.tty - cmd.SysProcAttr.Setctty = true - cmd.SysProcAttr.Setsid = true + // Pass PTY and TTY to child as well since a terminal is attached. + t.cmd.ExtraFiles = append(t.cmd.ExtraFiles, t.pty) + t.cmd.ExtraFiles = append(t.cmd.ExtraFiles, t.tty) - err = cmd.Start() + // Start the process. + err = t.cmd.Start() if err != nil { return trace.Wrap(err) } + // Save off the PID of the Teleport process under which the shell is executing. + t.pid = t.cmd.Process.Pid + return nil } @@ -204,6 +216,12 @@ func (t *terminal) Wait() (*ExecResult, error) { }, nil } +// Continue will resume execution of the process after it completes its +// pre-processing routine (placed in a cgroup). +func (t *terminal) Continue() { + t.ctx.contw.Close() +} + // Kill will force kill the terminal. func (t *terminal) Kill() error { if t.cmd.Process != nil { @@ -227,6 +245,11 @@ func (t *terminal) TTY() *os.File { return t.tty } +// PID returns the PID of the Teleport process that was re-execed. +func (t *terminal) PID() int { + return t.pid +} + // Close will free resources associated with the terminal. func (t *terminal) Close() error { var err error @@ -492,6 +515,11 @@ func (t *remoteTerminal) Wait() (*ExecResult, error) { }, nil } +// Continue does nothing for remote command execution. +func (r *remoteTerminal) Continue() { + return +} + func (t *remoteTerminal) Kill() error { err := t.session.Signal(ssh.SIGKILL) if err != nil { @@ -509,6 +537,12 @@ func (t *remoteTerminal) TTY() *os.File { return nil } +// PID returns the PID of the Teleport process that was re-execed. Always +// returns 0 for remote terminals. +func (t *remoteTerminal) PID() int { + return 0 +} + func (t *remoteTerminal) Close() error { // this closes the underlying stdin,stdout,stderr which is what ptyBuffer is // hooked to directly diff --git a/lib/srv/termhandlers.go b/lib/srv/termhandlers.go index 061a7f23f0c..57d461e5f47 100644 --- a/lib/srv/termhandlers.go +++ b/lib/srv/termhandlers.go @@ -18,10 +18,7 @@ package srv import ( "golang.org/x/crypto/ssh" - "io/ioutil" - "github.com/gravitational/teleport" - "github.com/gravitational/teleport/lib/pam" rsession "github.com/gravitational/teleport/lib/session" "github.com/gravitational/teleport/lib/sshutils" "github.com/gravitational/trace" @@ -37,82 +34,21 @@ type TermHandlers struct { // without a TTY. Result of execution is propagated back on the ExecResult // channel of the context. func (t *TermHandlers) HandleExec(ch ssh.Channel, req *ssh.Request, ctx *ServerContext) error { - execRequest, err := parseExecRequest(req, ctx) + // Save the request within the context. + ctx.request = req + + // Parse the exec request and store it in the context. + _, err := parseExecRequest(req, ctx) if err != nil { return trace.Wrap(err) } - // a terminal has been previously allocate for this command. - // run this inside an interactive session + // If a terminal was previously allocated for this command, run command in + // an interactive session. Otherwise run it in an exec session. if ctx.GetTerm() != nil { return t.SessionRegistry.OpenSession(ch, req, ctx) } - - // If this code is running on a Teleport node and PAM is enabled, then open a - // PAM context. - var pamContext *pam.PAM - if ctx.srv.Component() == teleport.ComponentNode { - conf, err := t.SessionRegistry.srv.GetPAM() - if err != nil { - return trace.Wrap(err) - } - - if conf.Enabled == true { - // Note, stdout/stderr is discarded here, otherwise MOTD would be printed to - // the users screen during exec requests. - pamContext, err = pam.Open(&pam.Config{ - ServiceName: conf.ServiceName, - Username: ctx.Identity.Login, - Stdin: ch, - Stderr: ioutil.Discard, - Stdout: ioutil.Discard, - }) - if err != nil { - return trace.Wrap(err) - } - ctx.Debugf("Opening PAM context for exec request %q.", execRequest.GetCommand()) - } - } - - // otherwise, regular execution - result, err := execRequest.Start(ch) - if err != nil { - return trace.Wrap(err) - } - // if the program failed to start, we should send that result back - if result != nil { - ctx.Debugf("Exec request (%v) result: %v", execRequest, result) - ctx.SendExecResult(*result) - } - - // in case if result is nil and no error, this means that program is - // running in the background - go func() { - result = execRequest.Wait() - if result != nil { - ctx.SendExecResult(*result) - } - - // If this code is running on a Teleport node and PAM is enabled, close the context. - if ctx.srv.Component() == teleport.ComponentNode { - conf, err := t.SessionRegistry.srv.GetPAM() - if err != nil { - ctx.Errorf("Unable to get PAM configuration from server: %v", err) - return - } - - if conf.Enabled == true { - err = pamContext.Close() - if err != nil { - ctx.Errorf("Unable to close PAM context for exec request: %q: %v", execRequest.GetCommand(), err) - return - } - ctx.Debugf("Closing PAM context for exec request: %q.", execRequest.GetCommand()) - } - } - }() - - return nil + return t.SessionRegistry.OpenExecSession(ch, req, ctx) } // HandlePTYReq handles requests of type "pty-req" which allocate a TTY for @@ -145,6 +81,7 @@ func (t *TermHandlers) HandlePTYReq(ch ssh.Channel, req *ssh.Request, ctx *Serve return trace.Wrap(err) } ctx.SetTerm(term) + ctx.termAllocated = true } term.SetWinSize(*params) term.SetTermType(ptyRequest.Env) @@ -163,12 +100,14 @@ func (t *TermHandlers) HandlePTYReq(ch ssh.Channel, req *ssh.Request, ctx *Serve func (t *TermHandlers) HandleShell(ch ssh.Channel, req *ssh.Request, ctx *ServerContext) error { var err error - // creating an empty exec request implies a interactive shell was requested + // Save the request within the context. + ctx.request = req + + // Creating an empty exec request implies a interactive shell was requested. ctx.ExecRequest, err = NewExecRequest(ctx, "") if err != nil { return trace.Wrap(err) } - if err := t.SessionRegistry.OpenSession(ch, req, ctx); err != nil { return trace.Wrap(err) } diff --git a/lib/utils/kernel.go b/lib/utils/kernel.go new file mode 100644 index 00000000000..097a8d0d3a9 --- /dev/null +++ b/lib/utils/kernel.go @@ -0,0 +1,48 @@ +/* +Copyright 2019 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "io/ioutil" + "runtime" + + "github.com/gravitational/teleport" + + "github.com/gravitational/trace" + + "github.com/coreos/go-semver/semver" +) + +// KernelVersion returns the kernel version of the host. This only returns +// something meaningful on Linux. +func KernelVersion() (*semver.Version, error) { + if runtime.GOOS != teleport.LinuxOS { + return nil, trace.BadParameter("requested kernel version on non-Linux host") + } + + buf, err := ioutil.ReadFile("/proc/sys/kernel/osrelease") + if err != nil { + return nil, trace.Wrap(err) + } + + ver, err := semver.NewVersion(string(buf)) + if err != nil { + return nil, trace.Wrap(err) + } + + return ver, nil +} diff --git a/lib/web/apiserver_test.go b/lib/web/apiserver_test.go index b54b1d07b16..c2d3ff2540b 100644 --- a/lib/web/apiserver_test.go +++ b/lib/web/apiserver_test.go @@ -46,6 +46,7 @@ import ( "github.com/gravitational/teleport/lib/auth" "github.com/gravitational/teleport/lib/auth/mocku2f" "github.com/gravitational/teleport/lib/backend" + "github.com/gravitational/teleport/lib/bpf" "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/events" @@ -57,6 +58,7 @@ import ( "github.com/gravitational/teleport/lib/secret" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/session" + "github.com/gravitational/teleport/lib/srv" "github.com/gravitational/teleport/lib/srv/regular" "github.com/gravitational/teleport/lib/sshutils" "github.com/gravitational/teleport/lib/utils" @@ -103,6 +105,21 @@ var _ = Suite(&WebSuite{ clock: clockwork.NewFakeClock(), }) +// TestMain will re-execute Teleport to run a command if "exec" is passed to +// it as an argument. Otherwise it will run tests as normal. +func TestMain(m *testing.M) { + // If the test is re-executing itself, execute the command that comes over + // the pipe. + if len(os.Args) == 2 && os.Args[1] == teleport.ExecSubCommand { + srv.RunCommand() + return + } + + // Otherwise run tests as normal. + code := m.Run() + os.Exit(code) +} + func (s *WebSuite) SetUpSuite(c *C) { var err error os.Unsetenv(teleport.DebugEnvVar) @@ -171,6 +188,7 @@ func (s *WebSuite) SetUpTest(c *C) { regular.SetSessionServer(nodeClient), regular.SetAuditLog(nodeClient), regular.SetPAMConfig(&pam.Config{Enabled: false}), + regular.SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) s.node = node @@ -217,6 +235,7 @@ func (s *WebSuite) SetUpTest(c *C) { regular.SetSessionServer(s.proxyClient), regular.SetAuditLog(s.proxyClient), regular.SetNamespace(defaults.Namespace), + regular.SetBPF(&bpf.NOP{}), ) c.Assert(err, IsNil) diff --git a/metrics.go b/metrics.go index 1fbcaafa4d4..0d6e543fe19 100644 --- a/metrics.go +++ b/metrics.go @@ -120,6 +120,15 @@ const ( // MetricBackendBatchFailedReadRequests measures failed backend batch read requests count MetricBackendBatchFailedReadRequests = "backend_batch_read_requests_failed_total" + // MetricLostCommandEvents measures the number of command events that were lost + MetricLostCommandEvents = "bpf_lost_command_events" + + // MetricLostDiskEvents measures the number of disk events that were lost. + MetricLostDiskEvents = "bpf_lost_disk_events" + + // MetricLostNetworkEvents measures the number of network events that were lost. + MetricLostNetworkEvents = "bpf_lost_network_events" + // TagRange is a tag specifying backend requests TagRange = "range" diff --git a/tool/teleport/common/teleport.go b/tool/teleport/common/teleport.go index 518bf19e477..27ab096f6a3 100644 --- a/tool/teleport/common/teleport.go +++ b/tool/teleport/common/teleport.go @@ -29,6 +29,7 @@ import ( "github.com/gravitational/teleport/lib/config" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/service" + "github.com/gravitational/teleport/lib/srv" "github.com/gravitational/teleport/lib/sshutils/scp" "github.com/gravitational/teleport/lib/utils" @@ -63,13 +64,15 @@ func Run(options Options) (executedCommand string, conf *service.Config) { // define global flags: var ccf config.CommandLineFlags var scpFlags scp.Flags + var execDebug bool // define commands: start := app.Command("start", "Starts the Teleport service.") status := app.Command("status", "Print the status of the current SSH session.") dump := app.Command("configure", "Print the sample config file into stdout.") ver := app.Command("version", "Print the version.") - scpc := app.Command("scp", "server-side implementation of scp").Hidden() + scpc := app.Command("scp", "Server-side implementation of SCP.").Hidden() + exec := app.Command("exec", "Used internally by Teleport to re-exec itself.").Hidden() app.HelpFlag.Short('h') // define start flags: @@ -137,6 +140,9 @@ func Run(options Options) (executedCommand string, conf *service.Config) { scpc.Flag("local-addr", "local address which accepted the request").StringVar(&scpFlags.LocalAddr) scpc.Arg("target", "").StringsVar(&scpFlags.Target) + // Define flags for the "exec" subcommand. + exec.Flag("debug", "Debug mode").Short('d').Default("false").BoolVar(&execDebug) + // parse CLI commands+flags: command, err := app.Parse(options.Args) if err != nil { @@ -167,6 +173,8 @@ func Run(options Options) (executedCommand string, conf *service.Config) { err = onStatus() case dump.FullCommand(): onConfigDump() + case exec.FullCommand(): + err = onExec(execDebug) case ver.FullCommand(): utils.PrintVersion() } @@ -258,6 +266,12 @@ func onSCP(scpFlags *scp.Flags) (err error) { return trace.Wrap(cmd.Execute(&StdReadWriter{})) } +// onExec will re-execute Teleport. +func onExec(debug bool) error { + srv.RunCommand() + return nil +} + type StdReadWriter struct { } diff --git a/version.go b/version.go index d9aaa602d68..c944aab8504 100644 --- a/version.go +++ b/version.go @@ -3,7 +3,7 @@ package teleport const ( - Version = "4.2.0-alpha.5" + Version = "4.2.0-dev.4" ) // Gitref variable is automatically set to the output of git-describe