diff --git a/assets/loadtest/k8s/Makefile b/assets/loadtest/k8s/Makefile index 915636397cc..71f785b0b32 100644 --- a/assets/loadtest/k8s/Makefile +++ b/assets/loadtest/k8s/Makefile @@ -115,7 +115,7 @@ install-teleport: install-auth install-proxy install-node install-iot-node .PHONY: delete-teleport delete-teleport: delete-tc delete-nodes delete-proxy delete-auth -# installs grafana and influxdb +# installs grafana, influxdb, and prometheus .PHONY: install-monitor install-monitor: kubectl create configmap grafana-config -n loadtest \ @@ -128,13 +128,16 @@ install-monitor: kubectl apply -f influxdb.yaml @make expand-yaml FILENAME=grafana kubectl apply -f grafana-gen.yaml + @make expand-yaml FILENAME=prometheus + kubectl apply -f prometheus-gen.yaml -# deletes grafana and influxdb deployments, services and configmaps +# deletes grafana, influxdb, and prometheus deployments, services and configmaps .PHONY: delete-monitor delete-monitor: kubectl delete -f influxdb.yaml --ignore-not-found kubectl delete -f grafana-gen.yaml --ignore-not-found kubectl delete configmap grafana-config -n loadtest --ignore-not-found + kubectl delete -f prometheus-gen.yaml --ignore-not-found # installs an etcd cluster .PHONY: install-etcd @@ -460,4 +463,4 @@ fetch-profiles: # output file will be named the same with a -gen suffix, i.e input = test then output will be test-gen.yaml .PHONY: expand-yaml expand-yaml: - @bash -c "set -a && source ./secrets/secrets.env && set +a && envsubst < $(FILENAME).yaml > $(FILENAME)-gen.yaml" \ No newline at end of file + @bash -c "set -a && source ./secrets/secrets.env && set +a && envsubst < $(FILENAME).yaml > $(FILENAME)-gen.yaml" diff --git a/assets/loadtest/k8s/auth-etcd.yaml b/assets/loadtest/k8s/auth-etcd.yaml index 15d648055ec..049df023b70 100644 --- a/assets/loadtest/k8s/auth-etcd.yaml +++ b/assets/loadtest/k8s/auth-etcd.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: teleport-role: auth + backend: etcd + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config diff --git a/assets/loadtest/k8s/auth-firestore.yaml b/assets/loadtest/k8s/auth-firestore.yaml index a3acd6d26cf..3b5c3c3affb 100644 --- a/assets/loadtest/k8s/auth-firestore.yaml +++ b/assets/loadtest/k8s/auth-firestore.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: teleport-role: auth + backend: firestore + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config diff --git a/assets/loadtest/k8s/prometheus.yaml b/assets/loadtest/k8s/prometheus.yaml new file mode 100644 index 00000000000..3649f3f7f92 --- /dev/null +++ b/assets/loadtest/k8s/prometheus.yaml @@ -0,0 +1,291 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: prometheus-loadtest +--- +# Source: prometheus/templates/server/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest + annotations: + {} +--- +# Source: prometheus/templates/server/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +data: + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + remote_write: + - url: ${PROM_REMOTE_URL} + basic_auth: + username: ${PROM_USER} + password: ${PROM_PASSWORD} + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: prometheus/templates/server/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/templates/server/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server + namespace: prometheus-loadtest +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-server +--- +# Source: prometheus/templates/server/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + component: "server" + app: prometheus + release: prometheus + sessionAffinity: None + type: "ClusterIP" +--- +# Source: prometheus/templates/server/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +spec: + selector: + matchLabels: + component: "server" + app: prometheus + release: prometheus + replicas: 1 + template: + metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + spec: + enableServiceLinks: true + serviceAccountName: prometheus-server + containers: + - name: prometheus-server-configmap-reload + image: "jimmidyson/configmap-reload:v0.5.0" + imagePullPolicy: "IfNotPresent" + args: + - --volume-dir=/etc/config + - --webhook-url=http://127.0.0.1:9090/-/reload + resources: + {} + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "quay.io/prometheus/prometheus:v2.31.1" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + resources: + {} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + hostNetwork: false + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus-server + - name: storage-volume + emptyDir: + {} diff --git a/assets/loadtest/k8s/proxy.yaml b/assets/loadtest/k8s/proxy.yaml index 4ef2396e8cb..4fd0dac3381 100644 --- a/assets/loadtest/k8s/proxy.yaml +++ b/assets/loadtest/k8s/proxy.yaml @@ -14,6 +14,8 @@ spec: metadata: labels: teleport-role: proxy + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config @@ -100,4 +102,4 @@ spec: targetPort: 3036 protocol: TCP selector: - teleport-role: proxy \ No newline at end of file + teleport-role: proxy diff --git a/assets/loadtest/k8s/secrets/Makefile b/assets/loadtest/k8s/secrets/Makefile index c0110552f20..6151d28f801 100644 --- a/assets/loadtest/k8s/secrets/Makefile +++ b/assets/loadtest/k8s/secrets/Makefile @@ -23,6 +23,21 @@ env: exit 1; \ fi + @if [ -z ${PROM_REMOTE_URL} ]; then \ + echo "PROM_REMOTE_URL is not set, cannot apply cluster."; \ + exit 1; \ + fi + + @if [ -z ${PROM_USER} ]; then \ + echo "PROM_USER is not set, cannot apply cluster."; \ + exit 1; \ + fi + + @if [ -z ${PROM_PASSWORD} ]; then \ + echo "PROM_PASSWORD is not set, cannot apply cluster."; \ + exit 1; \ + fi + @echo PROXY_IP=$(shell make -C ../../network get-proxy-ip) > secrets.env @echo PROXY_HOST=${PROXY_HOST} >> secrets.env @echo GRAFANA_IP=$(shell make -C ../../network get-grafana-ip) >> secrets.env @@ -31,6 +46,9 @@ env: @echo PROXY_TOKEN=$(shell cat proxy-token) >> secrets.env @echo TC_TOKEN=$(shell cat tc-token) >> secrets.env @echo GCP_PROJECT=$(shell make -C ../../cluster get-project) >> secrets.env + @echo PROM_REMOTE_URL=${PROM_REMOTE_URL} >> secrets.env + @echo PROM_USER=${PROM_USER} >> secrets.env + @echo PROM_PASSWORD=${PROM_PASSWORD} >> secrets.env grafana-pass: openssl rand -base64 32 | tr -d '\n' > grafana-pass @@ -56,4 +74,4 @@ join-tokens: node-token proxy-token tc-token # removes everything .PHONY:clean clean: - rm -rf *-pass *-token *-auth *.env \ No newline at end of file + rm -rf *-pass *-token *-auth *.env