Add agentless installer in the teleport discovery service (#19648)

* Add agentless installer

* Resolve comments

* Resolve comments

* Use GetCertAuthorities locally

* Try to get IMDS hostname

* Try get imds hostname first

This seems to be how its implemented for non-agentless nodes

* Use FIPS cipher suites

* use the openssh ca, resolve comments

* write keys to /etc/teleport/agentless by default

* Resolve comment

* lints

* test fixes

api/types/installers/agentless-installer.sh.tmpl

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+
+(
+  flock -n 9 || exit 1
+  if grep -q "Section created by 'teleport join openssh'" "$SSHD_CONFIG"; then
+    exit 0
+  fi
+
+  . /etc/os-release
+
+  if [ "$ID" = "debian" ] || [ "$ID" = "ubuntu" ]; then
+    # old versions of ubuntu require that keys get added by `apt-key add`, without
+    # adding the key apt shows a key signing error when installing teleport.
+    if [ "$VERSION_CODENAME" = "xenial" ] || [ "$VERSION_CODENAME" = "trusty" ]; then
+      curl -o /tmp/teleport-pubkey.asc https://deb.releases.teleport.dev/teleport-pubkey.asc
+      cat /tmp/teleport-pubkey.asc | sudo apt-key add -
+      echo "deb https://apt.releases.teleport.dev/ubuntu ${VERSION_CODENAME?} stable/{{ .MajorVersion }}" | sudo tee /etc/apt/sources.list.d/teleport.list
+      rm /tmp/teleport-pubkey.asc
+    else
+      curl https://deb.releases.teleport.dev/teleport-pubkey.asc | sudo tee /usr/share/keyrings/teleport-archive-keyring.asc
+      echo "deb [signed-by=/usr/share/keyrings/teleport-archive-keyring.asc]  https://apt.releases.teleport.dev/${ID?} ${VERSION_CODENAME?} stable/{{ .MajorVersion }}" | sudo tee /etc/apt/sources.list.d/teleport.list >/dev/null
+    fi
+    sudo apt-get update
+    sudo apt-get install -y teleport
+  elif [ "$ID" = "amzn" ] || [ "$ID" = "rhel" ]; then
+    if [ "$ID" = "rhel" ]; then
+      VERSION_ID=$(echo "$VERSION_ID" | sed 's/\..*//') # convert version numbers like '7.2' to only include the major version
+    fi
+    sudo yum-config-manager --add-repo \
+      "$(rpm --eval "https://yum.releases.teleport.dev/$ID/$VERSION_ID/Teleport/%{_arch}/stable/{{ .MajorVersion }}/teleport.repo")"
+    sudo yum install -y teleport
+  else
+    echo "Unsupported distro: $ID"
+    exit 1
+  fi
+
+  IMDS_TOKEN=$(curl -m5 -sS -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300")
+  LOCAL_IP=$(curl -m5 -sS -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" http://169.254.169.254/latest/meta-data/local-ipv4)
+  PUBLIC_IP=$(curl -m5 -sS -H "X-aws-ec2-metadata-token: ${IMDS_TOKEN}" http://169.254.169.254/latest/meta-data/public-ipv4 || echo "")
+
+  PRINCIPALS=""
+  if [ ! "$LOCAL_IP" = "" ]; then
+    PRINCIPALS="$LOCAL_IP,$PRINCIPALS"
+  fi
+  if [ ! "$PUBLIC_IP" = "" ]; then
+    PRINCIPALS="$PUBLIC_IP,$PRINCIPALS"
+  fi
+
+  sudo /usr/bin/teleport join openssh \
+    --openssh-config="${SSHD_CONFIG}" \
+    --join-method=iam \
+    --token="$1" \
+    --proxy-server="{{ .PublicProxyAddr }}" \
+    --additional-principals="$PRINCIPALS" \
+    --restart-sshd
+
+) 9>/var/lock/teleport_install.lock

api/types/installers/installers.go

@@ -25,14 +25,25 @@ import (
 //go:embed installer.sh.tmpl
 var defaultInstallScript string
 
+//go:embed agentless-installer.sh.tmpl
+var defaultAgentlessInstallScript string
+
 // InstallerScriptName is the name of the by default populated, EC2
 // installer script
 const InstallerScriptName = "default-installer"
 
+// InstallerScriptName is the name of the by default populated, EC2
+// installer script when agentless mode is enabled for a matcher
+const InstallerScriptNameAgentless = "default-agentless-installer"
+
 // DefaultInstaller represents a the default installer script provided
 // by teleport
 var DefaultInstaller = types.MustNewInstallerV1(InstallerScriptName, defaultInstallScript)
 
+// DefaultAgentlessInstaller represents a the default agentless installer script provided
+// by teleport
+var DefaultAgentlessInstaller = types.MustNewInstallerV1(InstallerScriptNameAgentless, defaultAgentlessInstallScript)
+
 // Template is used to fill proxy address and version information into
 // the installer script
 type Template struct {

lib/auth/grpcserver.go

@@ -4271,8 +4271,13 @@ func (g *GRPCServer) GetInstaller(ctx context.Context, req *types.ResourceReques
 	}
 	res, err := auth.GetInstaller(ctx, req.Name)
 	if err != nil {
-		if trace.IsNotFound(err) && req.Name == installers.InstallerScriptName {
-			return installers.DefaultInstaller, nil
+		if trace.IsNotFound(err) {
+			switch req.Name {
+			case installers.InstallerScriptName:
+				return installers.DefaultInstaller, nil
+			case installers.InstallerScriptNameAgentless:
+				return installers.DefaultAgentlessInstaller, nil
+			}
 		}
 		return nil, trace.Wrap(err)
 	}
@@ -4295,13 +4300,17 @@ func (g *GRPCServer) GetInstallers(ctx context.Context, _ *emptypb.Empty) (*type
 	}
 	var installersV1 []*types.InstallerV1
 	needDefault := true
+	needDefaultAgentless := true
 	for _, inst := range res {
 		instV1, ok := inst.(*types.InstallerV1)
 		if !ok {
 			return nil, trace.BadParameter("unsupported installer type %T", inst)
 		}
-		if inst.GetName() == installers.InstallerScriptName {
+		switch inst.GetName() {
+		case installers.InstallerScriptName:
 			needDefault = false
+		case installers.InstallerScriptNameAgentless:
+			needDefaultAgentless = false
 		}
 		installersV1 = append(installersV1, instV1)
 	}
@@ -4310,6 +4319,7 @@ func (g *GRPCServer) GetInstallers(ctx context.Context, _ *emptypb.Empty) (*type
 		return &types.InstallerV1List{
 			Installers: []*types.InstallerV1{
 				installers.DefaultInstaller,
+				installers.DefaultAgentlessInstaller,
 			},
 		}, nil
 	}
@@ -4317,6 +4327,9 @@ func (g *GRPCServer) GetInstallers(ctx context.Context, _ *emptypb.Empty) (*type
 	if needDefault {
 		installersV1 = append(installersV1, installers.DefaultInstaller)
 	}
+	if needDefaultAgentless {
+		installersV1 = append(installersV1, installers.DefaultAgentlessInstaller)
+	}
 
 	return &types.InstallerV1List{
 		Installers: installersV1,

lib/config/configuration.go

@@ -66,6 +66,8 @@ type CommandLineFlags struct {
 	AuthServerAddr []string
 	// --token flag
 	AuthToken string
+	// --join-method flag
+	JoinMethod string
 	// CAPins are the SKPI hashes of the CAs used to verify the Auth Server.
 	CAPins []string
 	// --listen-ip flag
@@ -164,6 +166,17 @@ type CommandLineFlags struct {
 	// DatabaseMySQLServerVersion is the MySQL server version reported to a client
 	// if the value cannot be obtained from the database.
 	DatabaseMySQLServerVersion string
+
+	// ProxyServer is the url of the proxy server to connect to
+	ProxyServer string
+	// OpenSSHConfigPath is the path of the file to write agentless configuration to
+	OpenSSHConfigPath string
+	// OpenSSHKeysPath is the path to write teleport keys and certs into
+	OpenSSHKeysPath string
+	// AdditionalPrincipals are a list of extra principals to include when generating host keys.
+	AdditionalPrincipals string
+	// RestartOpenSSH indicates whether openssh should be restarted or not.
+	RestartOpenSSH bool
 }
 
 // ReadConfigFile reads /etc/teleport.yaml (or whatever is passed via --config flag)
@@ -433,7 +446,9 @@ func ApplyFileConfig(fc *FileConfig, cfg *service.Config) error {
 	}
 
 	if fc.Discovery.Enabled() {
-		applyDiscoveryConfig(fc, cfg)
+		if err := applyDiscoveryConfig(fc, cfg); err != nil {
+			return trace.Wrap(err)
+		}
 	}
 
 	return nil
@@ -1150,20 +1165,21 @@ func applySSHConfig(fc *FileConfig, cfg *service.Config) (err error) {
 	return nil
 }
 
-func applyDiscoveryConfig(fc *FileConfig, cfg *service.Config) {
+func applyDiscoveryConfig(fc *FileConfig, cfg *service.Config) error {
 	cfg.Discovery.Enabled = fc.Discovery.Enabled()
 	for _, matcher := range fc.Discovery.AWSMatchers {
+		installParams, err := matcher.InstallParams.Parse()
+		if err != nil {
+			return trace.Wrap(err)
+		}
+
 		cfg.Discovery.AWSMatchers = append(cfg.Discovery.AWSMatchers,
 			services.AWSMatcher{
 				Types:   matcher.Types,
 				Regions: matcher.Regions,
 				Tags:    matcher.Tags,
-				Params: services.InstallerParams{
-					JoinMethod: matcher.InstallParams.JoinParams.Method,
-					JoinToken:  matcher.InstallParams.JoinParams.TokenName,
-					ScriptName: matcher.InstallParams.ScriptName,
-				},
-				SSM: &services.AWSSSM{DocumentName: matcher.SSM.DocumentName},
+				Params:  installParams,
+				SSM:     &services.AWSSSM{DocumentName: matcher.SSM.DocumentName},
 			})
 	}
 
@@ -1192,6 +1208,7 @@ func applyDiscoveryConfig(fc *FileConfig, cfg *service.Config) {
 		)
 	}
 
+	return nil
 }
 
 // applyKubeConfig applies file configuration for the "kubernetes_service" section.

lib/config/configuration_test.go

@@ -358,6 +358,7 @@ func TestConfigReading(t *testing.T) {
 							TokenName: "aws-discovery-iam-token",
 							Method:    "iam",
 						},
+						SSHDConfig: "/etc/ssh/sshd_config",
 						ScriptName: "default-installer",
 					},
 					SSM: AWSSSM{DocumentName: "TeleportDiscoveryInstaller"},
@@ -829,6 +830,17 @@ SREzU8onbBsjMg9QDiSf5oJLKvd/Ren+zGY7
 			"testKey": "testValue",
 		},
 	))
+
+	require.True(t, cfg.Discovery.Enabled)
+	require.Equal(t, cfg.Discovery.AWSMatchers[0].Regions, []string{"eu-central-1"})
+	require.Equal(t, cfg.Discovery.AWSMatchers[0].Types, []string{"ec2"})
+	require.Equal(t, cfg.Discovery.AWSMatchers[0].Params, services.InstallerParams{
+		InstallTeleport: true,
+		JoinMethod:      "iam",
+		JoinToken:       defaults.IAMInviteTokenName,
+		ScriptName:      "default-installer",
+		SSHDConfig:      defaults.SSHDConfigPath,
+	})
 }
 
 // TestApplyConfigNoneEnabled makes sure that if a section is not enabled,

lib/config/fileconf.go

@@ -512,13 +512,22 @@ func checkAndSetDefaultsForAWSMatchers(matcherInput []AWSMatcher) error {
 			matcher.Tags = map[string]apiutils.Strings{types.Wildcard: {types.Wildcard}}
 		}
 
+		var installParams services.InstallerParams
+		var err error
+
 		if matcher.InstallParams == nil {
 			matcher.InstallParams = &InstallParams{
 				JoinParams: JoinParams{
 					TokenName: defaults.IAMInviteTokenName,
 					Method:    types.JoinMethodIAM,
 				},
-				ScriptName: installers.InstallerScriptName,
+				ScriptName:      installers.InstallerScriptName,
+				InstallTeleport: "",
+				SSHDConfig:      defaults.SSHDConfigPath,
+			}
+			installParams, err = matcher.InstallParams.Parse()
+			if err != nil {
+				return trace.Wrap(err)
 			}
 		} else {
 			if method := matcher.InstallParams.JoinParams.Method; method == "" {
@@ -526,17 +535,35 @@ func checkAndSetDefaultsForAWSMatchers(matcherInput []AWSMatcher) error {
 			} else if method != types.JoinMethodIAM {
 				return trace.BadParameter("only IAM joining is supported for EC2 auto-discovery")
 			}
-			if token := matcher.InstallParams.JoinParams.TokenName; token == "" {
+
+			if matcher.InstallParams.JoinParams.TokenName == "" {
 				matcher.InstallParams.JoinParams.TokenName = defaults.IAMInviteTokenName
 			}
 
+			if matcher.InstallParams.SSHDConfig == "" {
+				matcher.InstallParams.SSHDConfig = defaults.SSHDConfigPath
+			}
+
+			installParams, err = matcher.InstallParams.Parse()
+			if err != nil {
+				return trace.Wrap(err)
+			}
+
 			if installer := matcher.InstallParams.ScriptName; installer == "" {
-				matcher.InstallParams.ScriptName = installers.InstallerScriptName
+				if installParams.InstallTeleport {
+					matcher.InstallParams.ScriptName = installers.InstallerScriptName
+				} else {
+					matcher.InstallParams.ScriptName = installers.InstallerScriptNameAgentless
+				}
 			}
 		}
 
 		if matcher.SSM.DocumentName == "" {
-			matcher.SSM.DocumentName = defaults.AWSInstallerDocument
+			if installParams.InstallTeleport {
+				matcher.SSM.DocumentName = defaults.AWSInstallerDocument
+			} else {
+				matcher.SSM.DocumentName = defaults.AWSAgentlessInstallerDocument
+			}
 		}
 	}
 	return nil
@@ -1505,6 +1532,32 @@ type InstallParams struct {
 	// ScriptName is the name of the teleport installer script
 	// resource for the EC2 instance to execute
 	ScriptName string `yaml:"script_name,omitempty"`
+	// InstallTeleport disables agentless discovery
+	InstallTeleport string `yaml:"install_teleport,omitempty"`
+	// SSHDConfig provides the path to write sshd configuration changes
+	SSHDConfig string `yaml:"sshd_config,omitempty"`
+}
+
+func (ip *InstallParams) Parse() (services.InstallerParams, error) {
+	install := services.InstallerParams{
+		JoinMethod:      ip.JoinParams.Method,
+		JoinToken:       ip.JoinParams.TokenName,
+		ScriptName:      ip.ScriptName,
+		InstallTeleport: true,
+		SSHDConfig:      ip.SSHDConfig,
+	}
+
+	if ip.InstallTeleport == "" {
+		return install, nil
+	}
+
+	var err error
+	install.InstallTeleport, err = apiutils.ParseBool(ip.InstallTeleport)
+	if err != nil {
+		return services.InstallerParams{}, trace.Wrap(err)
+	}
+
+	return install, nil
 }
 
 // AWSSSM provides options to use when executing SSM documents

lib/config/fileconf_test.go

@@ -868,6 +868,7 @@ func TestDiscoveryConfig(t *testing.T) {
 								TokenName: defaults.IAMInviteTokenName,
 								Method:    types.JoinMethodIAM,
 							},
+							SSHDConfig: "/etc/ssh/sshd_config",
 							ScriptName: installers.InstallerScriptName,
 						},
 						SSM: AWSSSM{DocumentName: defaults.AWSInstallerDocument},
@@ -914,6 +915,7 @@ func TestDiscoveryConfig(t *testing.T) {
 								TokenName: "hello-iam-a-token",
 								Method:    types.JoinMethodIAM,
 							},
+							SSHDConfig: "/etc/ssh/sshd_config",
 							ScriptName: "installer-custom",
 						},
 						SSM: AWSSSM{DocumentName: "hello_document"},
@@ -989,6 +991,7 @@ func TestDiscoveryConfig(t *testing.T) {
 								Method:    types.JoinMethodIAM,
 							},
 							ScriptName: installers.InstallerScriptName,
+							SSHDConfig: "/etc/ssh/sshd_config",
 						},
 					},
 				},

lib/config/testdata_test.go

@@ -189,6 +189,12 @@ kubernetes_service:
     kubeconfig_file: /tmp/kubeconfig
     labels:
       'testKey': 'testValue'
+
+discovery_service:
+    enabled: yes
+    aws:
+      - types: ["ec2"]
+        regions: ["eu-central-1"]
 `
 
 // NoServicesConfigString is a configuration file with no services enabled

lib/defaults/defaults.go

@@ -845,7 +845,16 @@ const (
 	// AWSInstallerDocument is the name of the default AWS document
 	// that will be called when executing the SSM command.
 	AWSInstallerDocument = "TeleportDiscoveryInstaller"
+
+	// AWSAgentlessInstallerDocument is the name of the default AWS document
+	// that will be called when executing the SSM command .
+	AWSAgentlessInstallerDocument = "TeleportAgentlessDiscoveryInstaller"
+
 	// IAMInviteTokenName is the name of the default Teleport IAM
 	// token to use when templating the script to be executed.
 	IAMInviteTokenName = "aws-discovery-iam-token"
+
+	// SSHDConfigPath is the path to the sshd config file to modify
+	// when using the agentless installer
+	SSHDConfigPath = "/etc/ssh/sshd_config"
 )

lib/services/matchers.go

@@ -60,6 +60,10 @@ type InstallerParams struct {
 	// ScriptName is the name of the teleport script for the EC2
 	// instance to execute
 	ScriptName string
+	// InstallTeleport disables agentless discovery
+	InstallTeleport bool
+	// SSHDConfig provides the path to write sshd configuration changes
+	SSHDConfig string
 }
 
 // AWSMatcher matches AWS databases.

lib/srv/discovery/discovery.go

@@ -344,7 +344,10 @@ func genInstancesLogStr[T any](instances []T, getID func(T) string) string {
 }
 
 func (s *Server) handleEC2Instances(instances *server.EC2Instances) error {
-	client, err := s.Clients.GetAWSSSMClient(instances.Region)
+	// TODO(amk): once agentless node inventory management is
+	//            implemented, create nodes after a successful SSM run
+
+	ec2Client, err := s.Clients.GetAWSSSMClient(instances.Region)
 	if err != nil {
 		return trace.Wrap(err)
 	}
@@ -355,9 +358,10 @@ func (s *Server) handleEC2Instances(instances *server.EC2Instances) error {
 
 	s.Log.Debugf("Running Teleport installation on these instances: AccountID: %s, Instances: %s",
 		instances.AccountID, genEC2InstancesLogStr(instances.Instances))
+
 	req := server.SSMRunRequest{
 		DocumentName: instances.DocumentName,
-		SSM:          client,
+		SSM:          ec2Client,
 		Instances:    instances.Instances,
 		Params:       instances.Parameters,
 		Region:       instances.Region,
@@ -379,12 +383,14 @@ func (s *Server) handleEC2Discovery() {
 			ec2Instances := instances.EC2Instances
 			s.Log.Debugf("EC2 instances discovered (AccountID: %s, Instances: %v), starting installation",
 				instances.AccountID, genEC2InstancesLogStr(ec2Instances.Instances))
+
 			if err := s.handleEC2Instances(ec2Instances); err != nil {
 				if trace.IsNotFound(err) {
 					s.Log.Debug("All discovered EC2 instances are already part of the cluster.")
 				} else {
 					s.Log.WithError(err).Error("Failed to enroll discovered EC2 instances.")
 				}
+
 			}
 		case <-s.ctx.Done():
 			s.ec2Watcher.Stop()

lib/srv/server/ec2_watcher.go

@@ -65,19 +65,24 @@ func NewEC2Watcher(ctx context.Context, matchers []services.AWSMatcher, clients
 		fetchInterval: time.Minute,
 		InstancesC:    make(chan Instances),
 	}
+
 	for _, matcher := range matchers {
 		for _, region := range matcher.Regions {
-			cl, err := clients.GetAWSEC2Client(region)
+			ec2Client, err := clients.GetAWSEC2Client(region)
 			if err != nil {
 				return nil, trace.Wrap(err)
 			}
+
 			fetcher := newEC2InstanceFetcher(ec2FetcherConfig{
 				Matcher:   matcher,
 				Region:    region,
 				Document:  matcher.SSM.DocumentName,
-				EC2Client: cl,
+				EC2Client: ec2Client,
 				Labels:    matcher.Tags,
 			})
+			if err != nil {
+				return nil, trace.Wrap(err)
+			}
 			watcher.fetchers = append(watcher.fetchers, fetcher)
 		}
 	}
@@ -116,16 +121,26 @@ func newEC2InstanceFetcher(cfg ec2FetcherConfig) *ec2InstanceFetcher {
 	} else {
 		log.Debug("Not setting any tag filters as there is a '*:...' tag present and AWS doesnt allow globbing on keys")
 	}
+	var parameters map[string]string
+	if cfg.Matcher.Params.InstallTeleport {
+		parameters = map[string]string{
+			"token":      cfg.Matcher.Params.JoinToken,
+			"scriptName": cfg.Matcher.Params.ScriptName,
+		}
+	} else {
+		parameters = map[string]string{
+			"token":          cfg.Matcher.Params.JoinToken,
+			"scriptName":     cfg.Matcher.Params.ScriptName,
+			"sshdConfigPath": cfg.Matcher.Params.SSHDConfig,
+		}
+	}
 
 	fetcherConfig := ec2InstanceFetcher{
 		EC2:          cfg.EC2Client,
 		Filters:      tagFilters,
 		Region:       cfg.Region,
 		DocumentName: cfg.Document,
-		Parameters: map[string]string{
-			"token":      cfg.Matcher.Params.JoinToken,
-			"scriptName": cfg.Matcher.Params.ScriptName,
-		},
+		Parameters:   parameters,
 	}
 	return &fetcherConfig
 }

lib/srv/server/ec2_watcher_test.go

@@ -143,12 +143,18 @@ func TestEC2Watcher(t *testing.T) {
 	}
 	matchers := []services.AWSMatcher{
 		{
+			Params: services.InstallerParams{
+				InstallTeleport: true,
+			},
 			Types:   []string{"EC2"},
 			Regions: []string{"us-west-2"},
 			Tags:    map[string]utils.Strings{"teleport": {"yes"}},
 			SSM:     &services.AWSSSM{},
 		},
 		{
+			Params: services.InstallerParams{
+				InstallTeleport: true,
+			},
 			Types:   []string{"EC2"},
 			Regions: []string{"us-west-2"},
 			Tags:    map[string]utils.Strings{"env": {"dev"}},

lib/srv/server/ssm_install.go

@@ -29,7 +29,6 @@ import (
 	"github.com/gravitational/trace"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/gravitational/teleport/api/types/events"
 	apievents "github.com/gravitational/teleport/api/types/events"
 	libevents "github.com/gravitational/teleport/lib/events"
 )
@@ -83,6 +82,7 @@ func (si *SSMInstaller) Run(ctx context.Context, req SSMRunRequest) error {
 	for k, v := range req.Params {
 		params[k] = []*string{aws.String(v)}
 	}
+
 	output, err := req.SSM.SendCommandWithContext(ctx, &ssm.SendCommandInput{
 		DocumentName: aws.String(req.DocumentName),
 		InstanceIds:  aws.StringSlice(ids),
@@ -144,8 +144,8 @@ func (si *SSMInstaller) checkCommand(ctx context.Context, req SSMRunRequest, com
 	if exitCode == 0 && code == libevents.SSMRunFailCode {
 		exitCode = -1
 	}
-	event := events.SSMRun{
-		Metadata: events.Metadata{
+	event := apievents.SSMRun{
+		Metadata: apievents.Metadata{
 			Type: libevents.SSMRunEvent,
 			Code: code,
 		},

tool/teleport/common/teleport.go

@@ -18,9 +18,12 @@ package common
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
+	"os/exec"
 	"os/user"
 	"path"
 	"path/filepath"
@@ -29,10 +32,18 @@ import (
 
 	"github.com/gravitational/kingpin"
 	"github.com/gravitational/trace"
+	"github.com/hashicorp/go-uuid"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ssh"
 
 	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/client/proto"
 	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/lib/auth"
+	"github.com/gravitational/teleport/lib/auth/authclient"
+	"github.com/gravitational/teleport/lib/auth/native"
+	"github.com/gravitational/teleport/lib/client"
+	"github.com/gravitational/teleport/lib/cloud/aws"
 	"github.com/gravitational/teleport/lib/config"
 	awsconfigurators "github.com/gravitational/teleport/lib/configurators/aws"
 	"github.com/gravitational/teleport/lib/defaults"
@@ -40,6 +51,7 @@ import (
 	"github.com/gravitational/teleport/lib/service"
 	"github.com/gravitational/teleport/lib/srv"
 	"github.com/gravitational/teleport/lib/sshutils/scp"
+	"github.com/gravitational/teleport/lib/tlsca"
 	"github.com/gravitational/teleport/lib/utils"
 )
 
@@ -52,6 +64,8 @@ type Options struct {
 	InitOnly bool
 }
 
+const agentlessKeysDir = "/etc/teleport/agentless"
+
 // Run inits/starts the process according to the provided options
 func Run(options Options) (app *kingpin.Application, executedCommand string, conf *service.Config) {
 	var err error
@@ -84,6 +98,8 @@ func Run(options Options) (app *kingpin.Application, executedCommand string, con
 	status := app.Command("status", "Print the status of the current SSH session.")
 	dump := app.Command("configure", "Generate a simple config file to get started.")
 	ver := app.Command("version", "Print the version of your teleport binary.")
+	join := app.Command("join", "Join a Teleport cluster without running the Teleport daemon")
+	joinOpenSSH := join.Command("openssh", "Join an SSH server to a Teleport cluster")
 	scpc := app.Command("scp", "Server-side implementation of SCP.").Hidden()
 	sftp := app.Command("sftp", "Server-side implementation of SFTP.").Hidden()
 	exec := app.Command(teleport.ExecSubCommand, "Used internally by Teleport to re-exec itself to run a command.").Hidden()
@@ -388,6 +404,17 @@ func Run(options Options) (app *kingpin.Application, executedCommand string, con
 	kubeState := app.Command("kube-state", "Used internally by Teleport to operate Kubernetes Secrets where Teleport stores its state.").Hidden()
 	kubeStateDelete := kubeState.Command("delete", "Used internally to delete Kubernetes states when the helm chart is uninstalled.").Hidden()
 
+	// teleport join --proxy-server=proxy.example.com --token=aws-join-token [--openssh-config=/path/to/sshd.conf] [--restart-sshd=true]
+	joinOpenSSH.Flag("proxy-server", "Address of the proxy server.").StringVar(&ccf.ProxyServer)
+	joinOpenSSH.Flag("token", "Invitation token to register with an auth server.").StringVar(&ccf.AuthToken)
+	joinOpenSSH.Flag("join-method", "Method to use to join the cluster (token, iam, ec2)").EnumVar(&ccf.JoinMethod, "token", "iam", "ec2")
+	joinOpenSSH.Flag("openssh-config", "Path to the OpenSSH config file").Default("/etc/ssh/sshd_config").StringVar(&ccf.OpenSSHConfigPath)
+	joinOpenSSH.Flag("openssh-keys-path", "Path to the place teleport keys and certs").Default(agentlessKeysDir).StringVar(&ccf.OpenSSHKeysPath)
+	joinOpenSSH.Flag("restart-sshd", "Restart OpenSSH").Default("true").BoolVar(&ccf.RestartOpenSSH)
+	joinOpenSSH.Flag("insecure", "Insecure mode disables certificate validation").BoolVar(&ccf.InsecureMode)
+	joinOpenSSH.Flag("additional-principals", "Comma separated list of host names the node can be accessed by").StringVar(&ccf.AdditionalPrincipals)
+	joinOpenSSH.Flag("debug", "Enable verbose logging to stderr.").Short('d').BoolVar(&ccf.Debug)
+
 	// parse CLI commands+flags:
 	utils.UpdateAppUsageTemplate(app, options.Args)
 	command, err := app.Parse(options.Args)
@@ -465,6 +492,8 @@ func Run(options Options) (app *kingpin.Application, executedCommand string, con
 	case discoveryBootstrapCmd.FullCommand():
 		configureDiscoveryBootstrapFlags.config.DiscoveryService = true
 		err = onConfigureDiscoveryBootstrap(configureDiscoveryBootstrapFlags)
+	case joinOpenSSH.FullCommand():
+		err = onJoinOpenSSH(ccf)
 	}
 	if err != nil {
 		utils.FatalError(err)
@@ -489,6 +518,320 @@ func OnStart(clf config.CommandLineFlags, config *service.Config) error {
 	return service.Run(context.TODO(), *config, nil)
 }
 
+// GenerateKeys generates TLS and SSH keypairs.
+func GenerateKeys() (private, sshpub, tlspub []byte, err error) {
+	privateKey, publicKey, err := native.GenerateKeyPair()
+	if err != nil {
+		return nil, nil, nil, trace.Wrap(err)
+	}
+
+	sshPrivateKey, err := ssh.ParseRawPrivateKey(privateKey)
+	if err != nil {
+		return nil, nil, nil, trace.Wrap(err)
+	}
+
+	tlsPublicKey, err := tlsca.MarshalPublicKeyFromPrivateKeyPEM(sshPrivateKey)
+	if err != nil {
+		return nil, nil, nil, trace.Wrap(err)
+	}
+
+	return privateKey, publicKey, tlsPublicKey, nil
+}
+
+func authenticatedUserClientFromIdentity(ctx context.Context, fips bool, proxy utils.NetAddr, id *auth.Identity) (auth.ClientI, error) {
+	var tlsConfig *tls.Config
+	var err error
+	var cipherSuites []uint16
+	if !fips {
+		cipherSuites = defaults.FIPSCipherSuites
+	}
+	tlsConfig, err = id.TLSConfig(cipherSuites)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	sshConfig, err := id.SSHClientConfig(fips)
+	if err != nil {
+		return nil, trace.Wrap(err)
+	}
+
+	authClientConfig := &authclient.Config{
+		TLS:         tlsConfig,
+		SSH:         sshConfig,
+		AuthServers: []utils.NetAddr{proxy},
+		Log:         log.StandardLogger(),
+	}
+
+	c, err := authclient.Connect(ctx, authClientConfig)
+	return c, trace.Wrap(err)
+}
+
+func getAWSInstanceHostname(ctx context.Context) (string, error) {
+	imds, err := aws.NewInstanceMetadataClient(ctx)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	hostname, err := imds.GetHostname(ctx)
+	if err != nil {
+		return "", trace.Wrap(err)
+	}
+	hostname = strings.ReplaceAll(hostname, " ", "_")
+	if utils.IsValidHostname(hostname) {
+		return hostname, nil
+	}
+	return "", trace.NotFound("failed to get a valid hostname from IMDS")
+}
+
+func tryCreateDefaultAgentlesKeysDir(agentlessKeysPath string) error {
+	baseTeleportDir := filepath.Dir(agentlessKeysPath)
+	_, err := os.Stat(baseTeleportDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Debugf("%s did not exist, creating %s", baseTeleportDir, agentlessKeysPath)
+			return trace.Wrap(os.MkdirAll(agentlessKeysPath, 0700))
+		}
+		return trace.Wrap(err)
+	}
+
+	var alreadyExistedAndDeleted bool
+	_, err = os.Stat(agentlessKeysPath)
+	if err == nil {
+		log.Debugf("%s already existed, removing old files", agentlessKeysPath)
+		err = os.RemoveAll(agentlessKeysPath)
+		if err != nil {
+			return trace.Wrap(err)
+		}
+		alreadyExistedAndDeleted = true
+	}
+
+	if os.IsNotExist(err) || alreadyExistedAndDeleted {
+		log.Debugf("%s did not exist, creating", agentlessKeysPath)
+		return trace.Wrap(os.Mkdir(agentlessKeysPath, 0700))
+	}
+
+	return trace.Wrap(err)
+}
+
+func onJoinOpenSSH(clf config.CommandLineFlags) error {
+	if err := checkSSHDConfigAlreadyUpdated(clf.OpenSSHConfigPath); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if clf.Debug {
+		log.SetLevel(log.DebugLevel)
+	}
+
+	addr, err := utils.ParseAddr(clf.ProxyServer)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	privateKey, sshPublicKey, tlsPublicKey, err := GenerateKeys()
+	if err != nil {
+		return trace.Wrap(err, "unable to generate new keypairs")
+	}
+
+	ctx := context.Background()
+	hostname, err := getAWSInstanceHostname(ctx)
+	if err != nil {
+		var hostErr error
+		hostname, hostErr = os.Hostname()
+		if hostErr != nil {
+			return trace.NewAggregate(err, hostErr)
+		}
+	}
+
+	// TODO(amk) get uuid from a cli argument once agentless inventory management is implemented to allow tsh ssh access via uuid
+	uuid, err := uuid.GenerateUUID()
+	_ = err
+
+	principals := []string{uuid}
+	for _, principal := range strings.Split(clf.AdditionalPrincipals, ",") {
+		if principal == "" {
+			continue
+		}
+		principals = append(principals, principal)
+	}
+
+	registerParams := auth.RegisterParams{
+		Token:                clf.AuthToken,
+		AdditionalPrincipals: principals,
+		JoinMethod:           types.JoinMethod(clf.JoinMethod),
+		ID: auth.IdentityID{
+			Role:     types.RoleNode,
+			NodeName: hostname,
+			HostUUID: uuid,
+		},
+		ProxyServer:        *addr,
+		PublicTLSKey:       tlsPublicKey,
+		PublicSSHKey:       sshPublicKey,
+		GetHostCredentials: client.HostCredentials,
+		FIPS:               clf.FIPS,
+	}
+
+	if clf.FIPS {
+		registerParams.CipherSuites = defaults.FIPSCipherSuites
+	}
+
+	certs, err := auth.Register(registerParams)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+
+	identity, err := auth.ReadIdentityFromKeyPair(privateKey, certs)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+
+	client, err := authenticatedUserClientFromIdentity(ctx, clf.FIPS, *addr, identity)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+
+	cas, err := client.GetCertAuthorities(ctx, types.OpenSSHCA, false)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+
+	var openSSHCA []byte
+	for _, ca := range cas {
+		for _, key := range ca.GetActiveKeys().SSH {
+			openSSHCA = append(openSSHCA, key.PublicKey...)
+			openSSHCA = append(openSSHCA, byte('\n'))
+		}
+	}
+
+	defaultKeysPath := clf.OpenSSHKeysPath == agentlessKeysDir
+	if defaultKeysPath {
+		if err := tryCreateDefaultAgentlesKeysDir(agentlessKeysDir); err != nil {
+			return trace.Wrap(err)
+		}
+	}
+
+	fmt.Printf("Writing Teleport keys to %s\n", clf.OpenSSHKeysPath)
+	if err := writeKeys(clf.OpenSSHKeysPath, privateKey, certs, openSSHCA); err != nil {
+		if defaultKeysPath {
+			rmdirErr := os.RemoveAll(agentlessKeysDir)
+			if rmdirErr != nil {
+				return trace.NewAggregate(err, rmdirErr)
+			}
+		}
+		return trace.Wrap(err)
+	}
+
+	fmt.Println("Updating OpenSSH config")
+	if err := updateSSHDConfig(clf.OpenSSHKeysPath, clf.OpenSSHConfigPath); err != nil {
+		return trace.Wrap(err)
+	}
+
+	fmt.Println("Restarting the OpenSSH daemon")
+	if err := restartSSHD(); err != nil {
+		return trace.Wrap(err)
+	}
+
+	return nil
+}
+
+const (
+	teleportKey       = "teleport"
+	teleportCert      = "teleport-cert.pub"
+	teleportOpenSSHCA = "teleport_user_ca.pub"
+)
+
+func writeKeys(sshdConfigDir string, private []byte, certs *proto.Certs, openSSHCA []byte) error {
+	if err := os.WriteFile(filepath.Join(sshdConfigDir, teleportKey), private, 0600); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if err := os.WriteFile(filepath.Join(sshdConfigDir, teleportCert), certs.SSH, 0600); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if err := os.WriteFile(filepath.Join(sshdConfigDir, teleportOpenSSHCA), openSSHCA, 0600); err != nil {
+		return trace.Wrap(err)
+	}
+
+	return nil
+}
+
+const sshdConfigSectionModificationHeader = "### Section created by 'teleport join openssh'"
+
+func checkSSHDConfigAlreadyUpdated(sshdConfigPath string) error {
+	contents, err := os.ReadFile(sshdConfigPath)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+
+	if strings.Contains(string(contents), sshdConfigSectionModificationHeader) {
+		return trace.AlreadyExists("not updating %s as it has already been modified by teleport", sshdConfigPath)
+	}
+	return nil
+}
+
+const sshdBinary = "sshd"
+
+func updateSSHDConfig(keyDir, sshdConfigPath string) error {
+	// has to write to the beginning of the sshd_config file as
+	// openssh takes the first occurrence of a setting
+	sshdConfig, err := os.OpenFile(sshdConfigPath, os.O_RDONLY|os.O_CREATE, 0644)
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	defer sshdConfig.Close()
+
+	configUpdate := fmt.Sprintf(`
+%s
+TrustedUserCaKeys %s
+HostKey %s
+HostCertificate %s
+### Section end
+`,
+		sshdConfigSectionModificationHeader,
+		filepath.Join(keyDir, "teleport_user_ca.pub"),
+		filepath.Join(keyDir, "teleport"),
+		filepath.Join(keyDir, "teleport-cert.pub"),
+	)
+	sshdConfigTmp, err := os.CreateTemp(keyDir, "")
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	defer sshdConfigTmp.Close()
+	if _, err := sshdConfigTmp.Write([]byte(configUpdate)); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if _, err := io.Copy(sshdConfigTmp, sshdConfig); err != nil {
+		return trace.Wrap(err)
+	}
+
+	if err := sshdConfigTmp.Sync(); err != nil {
+		return trace.Wrap(err)
+	}
+
+	cmd := exec.Command(sshdBinary, "-t", "-f", sshdConfigTmp.Name())
+	if err := cmd.Run(); err != nil {
+		return trace.Wrap(err, "teleport generated an invalid ssh config file, not writing")
+	}
+
+	if err := os.Rename(sshdConfigTmp.Name(), sshdConfigPath); err != nil {
+		return trace.Wrap(err)
+	}
+
+	return nil
+}
+
+func restartSSHD() error {
+	cmd := exec.Command("sshd", "-t")
+	if err := cmd.Run(); err != nil {
+		return trace.Wrap(err, "teleport generated an invalid ssh config file")
+	}
+
+	cmd = exec.Command("systemctl", "restart", "sshd")
+	if err := cmd.Run(); err != nil {
+		return trace.Wrap(err, "teleport failed to restart the sshd service")
+	}
+	return nil
+}
+
 // onStatus is the handler for "status" CLI command
 func onStatus() error {
 	sshClient := os.Getenv("SSH_CLIENT")