teleport/integration/proxy_helpers_test.go
2022-09-01 16:05:03 +00:00

605 lines
17 KiB
Go

/*
Copyright 2021 Gravitational, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package integration
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509/pkix"
"encoding/json"
"net"
"net/http"
"net/http/httptest"
"os/user"
"path/filepath"
"testing"
"time"
"github.com/google/uuid"
"github.com/gravitational/teleport/api/breaker"
"github.com/gravitational/teleport/api/types"
apiutils "github.com/gravitational/teleport/api/utils"
"github.com/gravitational/teleport/integration/helpers"
"github.com/gravitational/teleport/lib/defaults"
"github.com/gravitational/teleport/lib/service"
"github.com/gravitational/teleport/lib/services"
"github.com/gravitational/teleport/lib/srv/alpnproxy"
alpncommon "github.com/gravitational/teleport/lib/srv/alpnproxy/common"
"github.com/gravitational/teleport/lib/srv/db/postgres"
"github.com/gravitational/teleport/lib/tlsca"
"github.com/gravitational/teleport/lib/utils"
"github.com/gravitational/trace"
"github.com/jackc/pgconn"
"github.com/stretchr/testify/require"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/tools/clientcmd"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)
type ProxySuite struct {
root *helpers.TeleInstance
leaf *helpers.TeleInstance
}
type proxySuiteOptions struct {
rootConfigFunc func(suite *ProxySuite) *service.Config
leafConfigFunc func(suite *ProxySuite) *service.Config
rootConfigModFunc []func(config *service.Config)
leafConfigModFunc []func(config *service.Config)
rootClusterNodeName string
leafClusterNodeName string
rootClusterListeners helpers.InstanceListenerSetupFunc
leafClusterListeners helpers.InstanceListenerSetupFunc
rootTrustedSecretFunc func(suite *ProxySuite) []*helpers.InstanceSecrets
leafTrustedFunc func(suite *ProxySuite) []*helpers.InstanceSecrets
rootClusterRoles []types.Role
leafClusterRoles []types.Role
updateRoleMappingFunc func(t *testing.T, suite *ProxySuite)
trustedCluster types.TrustedCluster
}
func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite {
options := proxySuiteOptions{
rootClusterNodeName: Host,
leafClusterNodeName: Host,
rootClusterListeners: helpers.SingleProxyPortSetupOn(Host),
leafClusterListeners: helpers.SingleProxyPortSetupOn(Host),
}
for _, opt := range opts {
opt(&options)
}
rCfg := helpers.InstanceConfig{
ClusterName: "root.example.com",
HostID: uuid.New().String(),
NodeName: options.rootClusterNodeName,
Log: utils.NewLoggerForTests(),
}
rCfg.Listeners = options.rootClusterListeners(t, &rCfg.Fds)
rc := helpers.NewInstance(t, rCfg)
// Create leaf cluster.
lCfg := helpers.InstanceConfig{
ClusterName: "leaf.example.com",
HostID: uuid.New().String(),
NodeName: options.leafClusterNodeName,
Priv: rc.Secrets.PrivKey,
Pub: rc.Secrets.PubKey,
Log: utils.NewLoggerForTests(),
}
lCfg.Listeners = options.leafClusterListeners(t, &lCfg.Fds)
lc := helpers.NewInstance(t, lCfg)
suite := &ProxySuite{
root: rc,
leaf: lc,
}
user := mustGetCurrentUser(t)
for _, role := range options.rootClusterRoles {
rc.AddUserWithRole(user.Username, role)
}
for _, role := range options.leafClusterRoles {
lc.AddUserWithRole(user.Username, role)
}
rootTrustedSecrets := lc.Secrets.AsSlice()
if options.rootTrustedSecretFunc != nil {
rootTrustedSecrets = options.rootTrustedSecretFunc(suite)
}
rootConfig := options.rootConfigFunc(suite)
for _, v := range options.rootConfigModFunc {
v(rootConfig)
}
err := rc.CreateEx(t, rootTrustedSecrets, rootConfig)
require.NoError(t, err)
leafTrustedSecrets := rc.Secrets.AsSlice()
if options.leafTrustedFunc != nil {
leafTrustedSecrets = options.leafTrustedFunc(suite)
}
leafConfig := options.leafConfigFunc(suite)
for _, v := range options.leafConfigModFunc {
v(leafConfig)
}
err = lc.CreateEx(t, leafTrustedSecrets, leafConfig)
require.NoError(t, err)
require.NoError(t, rc.Start())
t.Cleanup(func() {
rc.StopAll()
})
require.NoError(t, lc.Start())
t.Cleanup(func() {
lc.StopAll()
})
if options.updateRoleMappingFunc != nil {
options.updateRoleMappingFunc(t, suite)
}
if options.trustedCluster != nil {
tryCreateTrustedCluster(t, suite.leaf.Process.GetAuthServer(), options.trustedCluster)
waitForTunnelConnections(t, suite.root.Process.GetAuthServer(), suite.leaf.Secrets.SiteName, 1)
}
return suite
}
func (p *ProxySuite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname string) {
nodeConfig := func() *service.Config {
tconf := service.MakeDefaultConfig()
tconf.Console = nil
tconf.Log = utils.NewLoggerForTests()
tconf.Hostname = tunnelNodeHostname
tconf.SetToken("token")
tconf.AuthServers = []utils.NetAddr{
{
AddrNetwork: "tcp",
Addr: p.leaf.Web,
},
}
tconf.Auth.Enabled = false
tconf.Proxy.Enabled = false
tconf.SSH.Enabled = true
tconf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
return tconf
}
_, err := p.leaf.StartNode(nodeConfig())
require.NoError(t, err)
// Wait for both cluster to see each other via reverse tunnels.
require.Eventually(t, waitForClusters(p.root.Tunnel, 1), 10*time.Second, 1*time.Second,
"Two clusters do not see each other: tunnels are not working.")
require.Eventually(t, waitForClusters(p.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
"Two clusters do not see each other: tunnels are not working.")
// Wait for both nodes to show up before attempting to dial to them.
err = waitForNodeCount(context.Background(), p.root, p.leaf.Secrets.SiteName, 2)
require.NoError(t, err)
}
func (p *ProxySuite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config helpers.ClientConfig) {
const (
deadline = time.Second * 5
nextIterWaitTime = time.Millisecond * 100
)
tc, err := p.root.NewClient(config)
require.NoError(t, err)
output := &bytes.Buffer{}
tc.Stdout = output
require.NoError(t, err)
cmd := []string{"echo", "hello world"}
err = utils.RetryStaticFor(deadline, nextIterWaitTime, func() error {
err = tc.SSH(context.TODO(), cmd, false)
return trace.Wrap(err)
})
require.NoError(t, err)
require.Equal(t, "hello world\n", output.String())
}
type proxySuiteOptionsFunc func(*proxySuiteOptions)
func withRootClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.rootClusterRoles = roles
}
}
func withLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.leafClusterRoles = roles
}
}
func withRootAndLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
withRootClusterRoles(roles...)(options)
withLeafClusterRoles(roles...)(options)
}
}
func withLeafClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.leafConfigFunc = fn
options.leafConfigModFunc = append(options.leafConfigModFunc, configModFunctions...)
}
}
func withRootClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.rootConfigFunc = fn
options.rootConfigModFunc = append(options.rootConfigModFunc, configModFunctions...)
}
}
func withRootAndLeafTrustedClusterReset() proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.rootTrustedSecretFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets {
return nil
}
options.leafTrustedFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets {
return nil
}
}
}
func withRootClusterNodeName(nodeName string) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.rootClusterNodeName = nodeName
}
}
func withLeafClusterNodeName(nodeName string) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.leafClusterNodeName = nodeName
}
}
func withRootClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.rootClusterListeners = fn
}
}
func withLeafClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.leafClusterListeners = fn
}
}
func newRole(t *testing.T, roleName string, username string) types.Role {
role, err := types.NewRoleV3(roleName, types.RoleSpecV5{
Allow: types.RoleConditions{
Logins: []string{username},
},
})
require.NoError(t, err)
return role
}
func rootClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config {
return func(suite *ProxySuite) *service.Config {
rc := suite.root
config := service.MakeDefaultConfig()
config.DataDir = t.TempDir()
config.Auth.Enabled = true
config.Auth.Preference.SetSecondFactor("off")
config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex)
config.Proxy.Enabled = true
config.Proxy.WebAddr.Addr = rc.Web
config.Proxy.DisableWebService = false
config.Proxy.DisableWebInterface = true
config.SSH.Enabled = true
config.SSH.Addr.Addr = rc.SSH
config.SSH.Labels = map[string]string{"env": "integration"}
config.CircuitBreakerConfig = breaker.NoopBreakerConfig()
return config
}
}
func leafClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config {
return func(suite *ProxySuite) *service.Config {
lc := suite.leaf
config := service.MakeDefaultConfig()
config.DataDir = t.TempDir()
config.Auth.Enabled = true
config.Auth.Preference.SetSecondFactor("off")
config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex)
config.Proxy.Enabled = true
config.Proxy.WebAddr.Addr = lc.Web
config.Proxy.DisableWebService = false
config.Proxy.DisableWebInterface = true
config.SSH.Enabled = true
config.SSH.Addr.Addr = lc.SSH
config.SSH.Labels = map[string]string{"env": "integration"}
config.CircuitBreakerConfig = breaker.NoopBreakerConfig()
return config
}
}
func createTestRole(username string) types.Role {
role := services.NewImplicitRole()
role.SetName("test")
role.SetLogins(types.Allow, []string{username})
role.SetNodeLabels(types.Allow, map[string]apiutils.Strings{"env": []string{"{{external.testing}}"}})
return role
}
func withStandardRoleMapping() proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) {
rc := suite.root
lc := suite.leaf
role := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
ca, err := lc.Process.GetAuthServer().GetCertAuthority(context.Background(), types.CertAuthID{
Type: types.UserCA,
DomainName: rc.Secrets.SiteName,
}, false)
require.NoError(t, err)
ca.SetRoles(nil) // Reset roles, otherwise they will take precedence.
ca.SetRoleMap(types.RoleMap{{Remote: role.GetName(), Local: []string{role.GetName()}}})
err = lc.Process.GetAuthServer().UpsertCertAuthority(ca)
require.NoError(t, err)
}
}
}
func withTrustedCluster() proxySuiteOptionsFunc {
return func(options *proxySuiteOptions) {
options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) {
root := suite.root
rootRole := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
secondRole := suite.leaf.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
trustedClusterToken := "trustedclustertoken"
err := root.Process.GetAuthServer().UpsertToken(context.Background(),
types.MustCreateProvisionToken(trustedClusterToken, []types.SystemRole{types.RoleTrustedCluster}, time.Time{}))
require.NoError(t, err)
trustedCluster := root.AsTrustedCluster(trustedClusterToken, types.RoleMap{
{Remote: rootRole.GetName(), Local: []string{secondRole.GetName()}},
})
err = trustedCluster.CheckAndSetDefaults()
require.NoError(t, err)
options.trustedCluster = trustedCluster
}
}
}
func mustGetCurrentUser(t *testing.T) *user.User {
user, err := user.Current()
require.NoError(t, err)
return user
}
func mustRunPostgresQuery(t *testing.T, client *pgconn.PgConn) {
result, err := client.Exec(context.Background(), "select 1").ReadAll()
require.NoError(t, err)
require.Equal(t, []*pgconn.Result{postgres.TestQueryResponse}, result)
}
func mustClosePostgresClient(t *testing.T, client *pgconn.PgConn) {
err := client.Close(context.Background())
require.NoError(t, err)
}
func k8ClientConfig(serverAddr, sni string) clientcmdapi.Config {
const clusterName = "gke_project_europecentral2a_cluster1"
return clientcmdapi.Config{
Clusters: map[string]*clientcmdapi.Cluster{
clusterName: {
Server: serverAddr,
InsecureSkipTLSVerify: true,
TLSServerName: sni,
},
},
Contexts: map[string]*clientcmdapi.Context{
clusterName: {
Cluster: clusterName,
AuthInfo: clusterName,
},
},
CurrentContext: clusterName,
}
}
func mkPodList() *v1.PodList {
return &v1.PodList{
TypeMeta: metav1.TypeMeta{
Kind: "PodList",
APIVersion: "v1",
},
Items: []v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Name: "firstcontainer-66b6c48dd-bqmwk",
},
},
},
}
}
func startKubeAPIMock(t *testing.T) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc("/apis/authorization.k8s.io/v1/selfsubjectaccessreviews", func(rw http.ResponseWriter, request *http.Request) {
})
mux.HandleFunc("/api/v1/namespaces/default/pods", func(rw http.ResponseWriter, request *http.Request) {
rw.Header().Add("Content-Type", "application/json")
err := json.NewEncoder(rw).Encode(mkPodList())
require.NoError(t, err)
})
svr := httptest.NewTLSServer(mux)
t.Cleanup(func() {
svr.Close()
})
return svr
}
func mustCreateKubeConfigFile(t *testing.T, config clientcmdapi.Config) string {
configPath := filepath.Join(t.TempDir(), "config.yaml")
err := clientcmd.WriteToFile(config, configPath)
require.NoError(t, err)
return configPath
}
func mustCreateListener(t *testing.T) net.Listener {
listener, err := net.Listen("tcp", ":0")
require.NoError(t, err)
t.Cleanup(func() {
listener.Close()
})
return listener
}
func mustStartALPNLocalProxy(t *testing.T, addr string, protocol alpncommon.Protocol) *alpnproxy.LocalProxy {
return mustStartALPNLocalProxyWithConfig(t, alpnproxy.LocalProxyConfig{
RemoteProxyAddr: addr,
Protocols: []alpncommon.Protocol{protocol},
InsecureSkipVerify: true,
})
}
func mustStartALPNLocalProxyWithConfig(t *testing.T, config alpnproxy.LocalProxyConfig) *alpnproxy.LocalProxy {
if config.Listener == nil {
config.Listener = mustCreateListener(t)
}
if config.SNI == "" {
address, err := utils.ParseAddr(config.RemoteProxyAddr)
require.NoError(t, err)
config.SNI = address.Host()
}
if config.ParentContext == nil {
config.ParentContext = context.TODO()
}
lp, err := alpnproxy.NewLocalProxy(config)
require.NoError(t, err)
t.Cleanup(func() {
lp.Close()
})
go func() {
err := lp.Start(context.Background())
require.NoError(t, err)
}()
return lp
}
func makeNodeConfig(nodeName, authAddr string) *service.Config {
nodeConfig := service.MakeDefaultConfig()
nodeConfig.Hostname = nodeName
nodeConfig.SetToken("token")
nodeConfig.AuthServers = []utils.NetAddr{
{
AddrNetwork: "tcp",
Addr: authAddr,
},
}
nodeConfig.Auth.Enabled = false
nodeConfig.Proxy.Enabled = false
nodeConfig.SSH.Enabled = true
nodeConfig.CircuitBreakerConfig = breaker.NoopBreakerConfig()
return nodeConfig
}
func mustCreateSelfSignedCert(t *testing.T) tls.Certificate {
caKey, caCert, err := tlsca.GenerateSelfSignedCA(pkix.Name{
CommonName: "localhost",
}, []string{"localhost"}, defaults.CATTL)
require.NoError(t, err)
cert, err := tls.X509KeyPair(caCert, caKey)
require.NoError(t, err)
return cert
}
// mockAWSALBProxy is a mock proxy server that simulates an AWS application
// load balancer where ALPN is not supported. Note that this mock does not
// actually balance traffic.
type mockAWSALBProxy struct {
net.Listener
proxyAddr string
cert tls.Certificate
}
func (m *mockAWSALBProxy) serve(ctx context.Context, t *testing.T) {
for {
select {
case <-ctx.Done():
return
default:
}
conn, err := m.Accept()
if err != nil {
if utils.IsOKNetworkError(err) {
return
}
require.NoError(t, err)
return
}
go func() {
// Handshake with incoming client and drops ALPN.
downstreamConn := tls.Server(conn, &tls.Config{
Certificates: []tls.Certificate{m.cert},
})
require.NoError(t, downstreamConn.HandshakeContext(ctx))
// Make a connection to the proxy server with ALPN protos.
upstreamConn, err := tls.Dial("tcp", m.proxyAddr, &tls.Config{
InsecureSkipVerify: true,
})
require.NoError(t, err)
utils.ProxyConn(ctx, downstreamConn, upstreamConn)
}()
}
}
func mustStartMockALBProxy(t *testing.T, proxyAddr string) *mockAWSALBProxy {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
m := &mockAWSALBProxy{
proxyAddr: proxyAddr,
Listener: mustCreateListener(t),
cert: mustCreateSelfSignedCert(t),
}
go m.serve(ctx, t)
return m
}