feature/add-etcd-vm-node-scrape (#614)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
- Enhanced system monitoring with a new configuration option to collect
etcd metrics. Users can now enable the scraping of etcd metrics via
updated settings, which improves observability.
- Introduced a secure proxy mechanism that conditionally routes metrics
data from etcd, offering administrators greater control over monitoring
capabilities.
- New configuration sections added to various bundles to support etcd
metrics scraping.
  
- **Bug Fixes**
- Removed outdated configuration for VMNodeScrape resource, ensuring
clarity and accuracy in monitoring configurations.

- **Chores**
- Added new service accounts, roles, and bindings to facilitate secure
access for monitoring etcd metrics.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: Andrei Kvapil <kvapss@gmail.com>

packages/core/platform/bundles/distro-full.yaml

@@ -82,6 +82,10 @@ releases:
   privileged: true
   optional: true
   dependsOn: [cilium,victoria-metrics-operator]
+  values:
+    scrapeRules:
+      etcd:
+        enabled: true
 
 - name: metallb
   releaseName: metallb

packages/core/platform/bundles/distro-hosted.yaml

@@ -58,6 +58,10 @@ releases:
   privileged: true
   optional: true
   dependsOn: [victoria-metrics-operator]
+  values:
+    scrapeRules:
+      etcd:
+        enabled: true
 
 - name: etcd-operator
   releaseName: etcd-operator

packages/core/platform/bundles/paas-full.yaml

@@ -103,6 +103,10 @@ releases:
   namespace: cozy-monitoring
   privileged: true
   dependsOn: [cilium,kubeovn,victoria-metrics-operator]
+  values:
+    scrapeRules:
+      etcd:
+        enabled: true
 
 - name: kubevirt-operator
   releaseName: kubevirt-operator

packages/core/platform/bundles/paas-hosted.yaml

@@ -70,6 +70,10 @@ releases:
   namespace: cozy-monitoring
   privileged: true
   dependsOn: [victoria-metrics-operator]
+  values:
+    scrapeRules:
+      etcd:
+        enabled: true
 
 - name: etcd-operator
   releaseName: etcd-operator

packages/system/monitoring-agents/templates/etcd-proxy-scrape.yaml

@@ -0,0 +1,138 @@
+{{- if .Values.scrapeRules.etcd.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-rbac-proxy
+  namespace: cozy-monitoring
+  labels:
+    app: kube-rbac-proxy
+spec:
+  selector:
+    matchLabels:
+      app: kube-rbac-proxy
+  template:
+    metadata:
+      labels:
+        app: kube-rbac-proxy
+    spec:
+      serviceAccountName: kube-rbac-proxy
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      containers:
+      - name: kube-rbac-proxy
+        image: quay.io/brancz/kube-rbac-proxy:v0.11.0
+        args:
+        - "--secure-listen-address=$(NODE_IP):9443"
+        - "--upstream=http://127.0.0.1:2381/"
+        env:
+        - name: NODE_IP
+          valueFrom:
+            fieldRef:
+              fieldPath: status.hostIP
+        ports:
+        - containerPort: 9443
+          name: etcd-metrics
+        securityContext:
+          runAsUser: 1000
+          runAsNonRoot: true
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-rbac-proxy
+  namespace: cozy-monitoring
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-rbac-proxy-auth
+rules:
+  - apiGroups: ["authentication.k8s.io"]
+    resources: ["tokenreviews"]
+    verbs: ["create"]
+  - apiGroups: ["authorization.k8s.io"]
+    resources: ["subjectaccessreviews"]
+    verbs: ["create"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-rbac-proxy-auth-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-rbac-proxy-auth
+subjects:
+  - kind: ServiceAccount
+    name: kube-rbac-proxy
+    namespace: cozy-monitoring
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: vm-scrape
+  namespace: cozy-monitoring
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: etcd-metrics-reader
+rules:
+- nonResourceURLs: ["/metrics"]
+  verbs: ["get"]
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: etcd-metrics-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: etcd-metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: vm-scrape
+  namespace: cozy-monitoring
+
+---
+
+apiVersion: v1
+kind: Secret
+type: kubernetes.io/service-account-token
+metadata:
+  name: vm-token
+  annotations:
+    kubernetes.io/service-account.name: vm-scrape
+
+---
+
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: etcd-managment-scrape
+spec:
+  podMetricsEndpoints:
+    - port: etcd-metrics
+      scheme: https
+      tlsConfig:
+        insecureSkipVerify: true
+      bearerTokenSecret:
+        name: vm-token
+        key: token
+  selector:
+    matchLabels:
+      app: kube-rbac-proxy
+{{- end }}

packages/system/monitoring-agents/templates/etcd-scrape.yaml

@@ -1,34 +0,0 @@
-#---
-#apiVersion: operator.victoriametrics.com/v1beta1
-#kind: VMNodeScrape
-#metadata:
-#  name: kube-etcd
-#  namespace: cozy-monitoring
-#spec:
-#  selector:
-#    node-role.kubernetes.io/control-plane: ""
-#  bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
-#  honorLabels: true
-#  metricRelabelConfigs:
-#  - action: labeldrop
-#    regex: (uid)
-#  - action: labeldrop
-#    regex: (id|name)
-#  - action: drop
-#    regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
-#    source_labels:
-#    - __name__
-#  port: "2379"
-#  relabelConfigs:
-#  - action: labelmap
-#    regex: __meta_kubernetes_node_label_(.+)
-#  - sourceLabels:
-#    - __metrics_path__
-#    targetLabel: metrics_path
-#  - replacement: etcd
-#    targetLabel: job
-#  scheme: https
-#  scrapeTimeout: 5s
-#  tlsConfig:
-#    caFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-#    insecureSkipVerify: true

packages/system/monitoring-agents/values.yaml

@@ -359,3 +359,7 @@ fluent-bit:
           Name modify
           Match *
           Add cluster root-cluster
+
+scrapeRules:
+  etcd:
+    enabled: false