From caadbca811e80c0677c592d67a54bbb926715255 Mon Sep 17 00:00:00 2001 From: Carson Anderson Date: Thu, 3 Mar 2022 10:42:55 -0700 Subject: [PATCH] Add prometheus metrics collection to load test (#10591) This adds a prometheus instance with remote write as part of the load test and persists results to remote prometheus target. Adds labels to auth pods to differentiate the backend being tested as well as annotations to indicate the pods should be scraped for metrics. --- assets/loadtest/k8s/Makefile | 9 +- assets/loadtest/k8s/auth-etcd.yaml | 3 + assets/loadtest/k8s/auth-firestore.yaml | 3 + assets/loadtest/k8s/prometheus.yaml | 291 ++++++++++++++++++++++++ assets/loadtest/k8s/proxy.yaml | 4 +- assets/loadtest/k8s/secrets/Makefile | 20 +- 6 files changed, 325 insertions(+), 5 deletions(-) create mode 100644 assets/loadtest/k8s/prometheus.yaml diff --git a/assets/loadtest/k8s/Makefile b/assets/loadtest/k8s/Makefile index 915636397cc..71f785b0b32 100644 --- a/assets/loadtest/k8s/Makefile +++ b/assets/loadtest/k8s/Makefile @@ -115,7 +115,7 @@ install-teleport: install-auth install-proxy install-node install-iot-node .PHONY: delete-teleport delete-teleport: delete-tc delete-nodes delete-proxy delete-auth -# installs grafana and influxdb +# installs grafana, influxdb, and prometheus .PHONY: install-monitor install-monitor: kubectl create configmap grafana-config -n loadtest \ @@ -128,13 +128,16 @@ install-monitor: kubectl apply -f influxdb.yaml @make expand-yaml FILENAME=grafana kubectl apply -f grafana-gen.yaml + @make expand-yaml FILENAME=prometheus + kubectl apply -f prometheus-gen.yaml -# deletes grafana and influxdb deployments, services and configmaps +# deletes grafana, influxdb, and prometheus deployments, services and configmaps .PHONY: delete-monitor delete-monitor: kubectl delete -f influxdb.yaml --ignore-not-found kubectl delete -f grafana-gen.yaml --ignore-not-found kubectl delete configmap grafana-config -n loadtest --ignore-not-found + kubectl delete -f prometheus-gen.yaml --ignore-not-found # installs an etcd cluster .PHONY: install-etcd @@ -460,4 +463,4 @@ fetch-profiles: # output file will be named the same with a -gen suffix, i.e input = test then output will be test-gen.yaml .PHONY: expand-yaml expand-yaml: - @bash -c "set -a && source ./secrets/secrets.env && set +a && envsubst < $(FILENAME).yaml > $(FILENAME)-gen.yaml" \ No newline at end of file + @bash -c "set -a && source ./secrets/secrets.env && set +a && envsubst < $(FILENAME).yaml > $(FILENAME)-gen.yaml" diff --git a/assets/loadtest/k8s/auth-etcd.yaml b/assets/loadtest/k8s/auth-etcd.yaml index 15d648055ec..049df023b70 100644 --- a/assets/loadtest/k8s/auth-etcd.yaml +++ b/assets/loadtest/k8s/auth-etcd.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: teleport-role: auth + backend: etcd + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config diff --git a/assets/loadtest/k8s/auth-firestore.yaml b/assets/loadtest/k8s/auth-firestore.yaml index a3acd6d26cf..3b5c3c3affb 100644 --- a/assets/loadtest/k8s/auth-firestore.yaml +++ b/assets/loadtest/k8s/auth-firestore.yaml @@ -14,6 +14,9 @@ spec: metadata: labels: teleport-role: auth + backend: firestore + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config diff --git a/assets/loadtest/k8s/prometheus.yaml b/assets/loadtest/k8s/prometheus.yaml new file mode 100644 index 00000000000..3649f3f7f92 --- /dev/null +++ b/assets/loadtest/k8s/prometheus.yaml @@ -0,0 +1,291 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: prometheus-loadtest +--- +# Source: prometheus/templates/server/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest + annotations: + {} +--- +# Source: prometheus/templates/server/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +data: + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + remote_write: + - url: ${PROM_REMOTE_URL} + basic_auth: + username: ${PROM_USER} + password: ${PROM_PASSWORD} + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_pod_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + recording_rules.yml: | + {} + rules: | + {} +--- +# Source: prometheus/templates/server/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/templates/server/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server + namespace: prometheus-loadtest +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-server +--- +# Source: prometheus/templates/server/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + component: "server" + app: prometheus + release: prometheus + sessionAffinity: None + type: "ClusterIP" +--- +# Source: prometheus/templates/server/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + name: prometheus-server + namespace: prometheus-loadtest +spec: + selector: + matchLabels: + component: "server" + app: prometheus + release: prometheus + replicas: 1 + template: + metadata: + labels: + component: "server" + app: prometheus + release: prometheus + chart: prometheus-15.4.0 + heritage: Helm + spec: + enableServiceLinks: true + serviceAccountName: prometheus-server + containers: + - name: prometheus-server-configmap-reload + image: "jimmidyson/configmap-reload:v0.5.0" + imagePullPolicy: "IfNotPresent" + args: + - --volume-dir=/etc/config + - --webhook-url=http://127.0.0.1:9090/-/reload + resources: + {} + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "quay.io/prometheus/prometheus:v2.31.1" + imagePullPolicy: "IfNotPresent" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + resources: + {} + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + hostNetwork: false + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus-server + - name: storage-volume + emptyDir: + {} diff --git a/assets/loadtest/k8s/proxy.yaml b/assets/loadtest/k8s/proxy.yaml index 4ef2396e8cb..4fd0dac3381 100644 --- a/assets/loadtest/k8s/proxy.yaml +++ b/assets/loadtest/k8s/proxy.yaml @@ -14,6 +14,8 @@ spec: metadata: labels: teleport-role: proxy + prometheus.io/scrape: "true" + prometheus.io/port: "3434" spec: volumes: - name: config @@ -100,4 +102,4 @@ spec: targetPort: 3036 protocol: TCP selector: - teleport-role: proxy \ No newline at end of file + teleport-role: proxy diff --git a/assets/loadtest/k8s/secrets/Makefile b/assets/loadtest/k8s/secrets/Makefile index c0110552f20..6151d28f801 100644 --- a/assets/loadtest/k8s/secrets/Makefile +++ b/assets/loadtest/k8s/secrets/Makefile @@ -23,6 +23,21 @@ env: exit 1; \ fi + @if [ -z ${PROM_REMOTE_URL} ]; then \ + echo "PROM_REMOTE_URL is not set, cannot apply cluster."; \ + exit 1; \ + fi + + @if [ -z ${PROM_USER} ]; then \ + echo "PROM_USER is not set, cannot apply cluster."; \ + exit 1; \ + fi + + @if [ -z ${PROM_PASSWORD} ]; then \ + echo "PROM_PASSWORD is not set, cannot apply cluster."; \ + exit 1; \ + fi + @echo PROXY_IP=$(shell make -C ../../network get-proxy-ip) > secrets.env @echo PROXY_HOST=${PROXY_HOST} >> secrets.env @echo GRAFANA_IP=$(shell make -C ../../network get-grafana-ip) >> secrets.env @@ -31,6 +46,9 @@ env: @echo PROXY_TOKEN=$(shell cat proxy-token) >> secrets.env @echo TC_TOKEN=$(shell cat tc-token) >> secrets.env @echo GCP_PROJECT=$(shell make -C ../../cluster get-project) >> secrets.env + @echo PROM_REMOTE_URL=${PROM_REMOTE_URL} >> secrets.env + @echo PROM_USER=${PROM_USER} >> secrets.env + @echo PROM_PASSWORD=${PROM_PASSWORD} >> secrets.env grafana-pass: openssl rand -base64 32 | tr -d '\n' > grafana-pass @@ -56,4 +74,4 @@ join-tokens: node-token proxy-token tc-token # removes everything .PHONY:clean clean: - rm -rf *-pass *-token *-auth *.env \ No newline at end of file + rm -rf *-pass *-token *-auth *.env