feat: add kps & grafana (#7)

2026-01-28 02:18:24 +00:00 · 2023-10-26 04:18:44 +08:00
parent 5c3c3b18da
commit 880e962501
15 changed files with 382 additions and 8 deletions
--- a/kube/clusters/biohazard/config/vars.sops.env
+++ b/kube/clusters/biohazard/config/vars.sops.env
@@ -69,6 +69,8 @@ APP_IP_RADOSGW=ENC[AES256_GCM,data:3ndMvS7qVTZxSg==,iv:n/5arRlOykLfrk8kGqPMaZegY
 APP_DNS_RGW_S3=ENC[AES256_GCM,data:X/DlP3vIFc07Sg==,iv:HlJ/AbTqCuOuszK8Lll8qsSNpuZOoty0lsnYCt1UF48=,tag:nFoxdgyYyZArPflmm2DwHQ==,type:str]
 APP_DNS_INGRESS_WILDCARD=ENC[AES256_GCM,data:aPYf3BwPvNA=,iv:Kgey2Z4+1JFa9JOOzG98QmBBMIp4fTPm8VPLw5d9gLw=,tag:R8Hb5kcuLFlIP0m1Aopdpg==,type:str]
 APP_DNS_HUBBLE=ENC[AES256_GCM,data:IcbmzSNwcLqbtg==,iv:qGuMNgCu39RMcdKjsGia8wCZ1Vpj8MVcDO2QQv4wONY=,tag:mqwjMLhKR4q0tjftCS25Lw==,type:str]
+APP_DNS_GRAFANA=ENC[AES256_GCM,data:1fP9SPrpsQs=,iv:1HhaHwRCW3tBV1cP81MiEpnNjy/TBl5WhbW0TRPvYp4=,tag:thnzc3N3mFANkqJpMPSwAg==,type:str]
+APP_DNS_PROMETHEUS=ENC[AES256_GCM,data:PoqEwDs/mFcp3d4=,iv:iPtCmwSqVGZ82PO+8jM0VfdlCfigyBd82rL5ytsPQxY=,tag:B9mwRNY5/Qht55m/sWyUYg==,type:str]
 APP_IP_KANIDM=ENC[AES256_GCM,data:VGm8gzd5D5x3phU=,iv:yS1pT2TSGKsTeFB0ouYUyTYEGD88d3DebpwSJ6lJpSs=,tag:kpa8wKJm4gdyCWKJ1A4n1w==,type:str]
 APP_UID_KANIDM=ENC[AES256_GCM,data:plVe/N8=,iv:sss67JiY8gaa0+UMs7rb1K+nDWP6BCKsnKuqj2txXSQ=,tag:exDjUeioDOBrkFQPF0tl+Q==,type:str]
 APP_DNS_KANIDM=ENC[AES256_GCM,data:Zthi8C9YcOVG,iv:NY8E+/Ij1w4Uq68bCfA7Fev5keEsg1uY100BvGDzCaE=,tag:wRGFWFr5wgGybwIB5EM4/Q==,type:str]
@@ -175,12 +177,12 @@ CONFIG_OVENMEDIAENGINE_NAME=ENC[AES256_GCM,data:58CuH8bcUHWXBZA=,iv:BN7x6aAJPbzI
 CONFIG_THELOUNGE_USERNAME=ENC[AES256_GCM,data:+C2aABtqq8YG,iv:4DYpguAvmaqPedRgrflDlKfX5jJEhyWXKuRS+UVgHLo=,tag:vfJko+R2D8ct7KZC2Vnujw==,type:str]
 CONFIG_THELOUNGE_JOIN=ENC[AES256_GCM,data:ocuC,iv:9Cn9zp2+iIVrEXYxklEtkpftmJwTGsWnff2xIG9KNec=,tag:3UL9Gn+kHoXu+40CFkP7sg==,type:str]
 CONFIG_PSONO_TITLE=ENC[AES256_GCM,data:ORXmkTqtuka3l5M0pdu1NKxdX3Pes3xdEMw=,iv:Mbw/KUQJcIdYdcWby6qeCY4Q31Vc+dUOjLLprHL5P9E=,tag:HavoGugubPrunCoOkL40Mw==,type:str]
-sops_pgp__list_0__map_created_at=2023-06-01T18:01:04Z
-sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFSXFvLzFQaFJ0OVJKUFV5\nTWh2OUltUlpJWFlVVytFYU9VajBHSnQ4SGdjCnRVbEVXdDVyUHJrR05Ba0xvUm1l\nTkt2YmNUZy90ZFA2b3QrODFKZ01EVG8KLS0tIEw2dkd1cnFCbnI5eWxKL2o1aDVB\nN0hveXZ2dWdxQ2k2L0pGR0ROMStVTmsK4dV/hNyDjsYnVUiFQ7kqdmcVHfYyVckz\nh/rwLjcZgsup72WDVP3v6Eul8B3LKFrSb8CDFA54tyQmSdFDCQC+Zg==\n-----END AGE ENCRYPTED FILE-----\n
-sops_mac=ENC[AES256_GCM,data:vZZXqsGwbu4cf7b9xjGgBgmUCOsTmojlbyEog6Ehx3xaNSuVK3YhMUxWNKrq6Mx3uOQm+vAEsWo6U1v20Jr94vdXM1ZkNYAg4tAfJyj2JDOJBKdcVe7KO1r1TRJky6Zk3fXifrGziSumZc1/CvfNTk7EHh81/qyCK00TWA+9s8E=,iv:/C1CMN7MayNpzLH/79778nolD72s0/JkmVR5wgvULcI=,tag:qsZ5JqDrBos5YHs3FXGNJQ==,type:str]
-sops_unencrypted_suffix=_unencrypted
-sops_version=3.7.3
+sops_lastmodified=2023-10-25T20:09:15Z
 sops_pgp__list_0__map_fp=31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2
-sops_lastmodified=2023-10-17T16:37:22Z
+sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFSXFvLzFQaFJ0OVJKUFV5\nTWh2OUltUlpJWFlVVytFYU9VajBHSnQ4SGdjCnRVbEVXdDVyUHJrR05Ba0xvUm1l\nTkt2YmNUZy90ZFA2b3QrODFKZ01EVG8KLS0tIEw2dkd1cnFCbnI5eWxKL2o1aDVB\nN0hveXZ2dWdxQ2k2L0pGR0ROMStVTmsK4dV/hNyDjsYnVUiFQ7kqdmcVHfYyVckz\nh/rwLjcZgsup72WDVP3v6Eul8B3LKFrSb8CDFA54tyQmSdFDCQC+Zg==\n-----END AGE ENCRYPTED FILE-----\n
+sops_mac=ENC[AES256_GCM,data:VXQjvZWeou9QXyEDVYmMv7/4NFdxKFfCT0M+159sMNYU2TUlxzsm+Cs1d1Hc//uXEEP+3VtFbc2CRYz5RZcTF7mj/ZCHZnlQTqNwHQOwoj+3f/hWl86lkcdwpsTyMVu3/0z5t9ox5J8NA3Wu67CXi1bpQgqJo+aWrW+kDGRB2xw=,iv:EgIV7xSA9NH2s8g7LbjzLwfC71L4IX9GaF9zbel16yA=,tag:X+9fh6fVkFvq31rV+1XoKg==,type:str]
+sops_version=3.7.3
 sops_age__list_0__map_recipient=age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj
+sops_unencrypted_suffix=_unencrypted
+sops_pgp__list_0__map_created_at=2023-06-01T18:01:04Z
 sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhF4DAAAAAAAAAAASAQdAbA35718t0WVKrjQFYUPviCb0lVuh8NpfSdJCHjHcWWww\n8ak4q4VL69tZLSjQHx+VsMmKooknxWz6pw0lGxyDYlZMQ81bodInjaZGFZSz8Uuh\n0l4BhDCNDBBALTrnTliz6/DAHvmavI4UxMHost5alFio9JPkTDNmXZyvcy1/R6aw\n/uhQXLUBRvm0TSOhBZb7d0SLkLfe02Um40w1TibpKXsZz1GOMbPRNBMHHra0QIuQ\n=0jA+\n-----END PGP MESSAGE-----\n
--- a/kube/clusters/biohazard/flux/kustomization.yaml
+++ b/kube/clusters/biohazard/flux/kustomization.yaml
@@ -28,9 +28,9 @@ resources:
  - ../../../deploy/core/ingress/external-proxy-x/
  - ../../../deploy/core/db/pg/
  - ../../../deploy/core/monitoring/metrics-server/
-  - ../../../deploy/core/monitoring/kube-state-metrics/
  - ../../../deploy/core/monitoring/node-exporter/
-  - ../../../deploy/core/monitoring/victoria/
+  - ../../../deploy/core/monitoring/kps/
+  - ../../../deploy/core/monitoring/grafana/
  - ../../../deploy/core/hardware/node-feature-discovery/
  - ../../../deploy/core/hardware/intel-device-plugins/
  - ../../../deploy/core/flux-system/
--- a/kube/deploy/core/monitoring/_deps/repo-grafana.yaml
+++ b/kube/deploy/core/monitoring/_deps/repo-grafana.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: grafana
+  namespace: flux-system
+spec:
+  interval: 1h
+  timeout: 3m0s
+  url: "https://grafana.github.io/helm-charts"
--- a/kube/deploy/core/monitoring/grafana/app/hr.yaml
+++ b/kube/deploy/core/monitoring/grafana/app/hr.yaml
@@ -0,0 +1,184 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.devbu.io/helmrelease_v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: grafana
+  namespace: monitoring
+spec:
+  chart:
+    spec:
+      chart: grafana
+      version: 6.61.1
+      sourceRef:
+        name: grafana
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    replicas: 3
+    env:
+      GF_EXPLORE_ENABLED: true
+      GF_LOG_MODE: console
+      GF_LOG_FILTERS: rendering:debug
+      GF_SECURITY_ALLOW_EMBEDDING: true
+      GF_SECURITY_COOKIE_SAMESITE: grafana
+      GF_DATE_FORMATS_USE_BROWSER_LOCALE: true
+    envFromSecret: grafana-secret
+    grafana.ini:
+      server:
+        root_url: "https://${APP_DNS_GRAFANA}"
+      analytics:
+        check_for_updates: false
+        check_for_plugin_updates: false
+        reporting_enabled: false
+      auth:
+        oauth_auto_login: true
+      auth.generic_oauth:
+        enabled: true
+        name: "JJGadgets Auth"
+        scopes: "openid profile email groups"
+        empty_scopes: false
+        login_attribute_path: preferred_username
+        groups_attribute_path: groups
+        name_attribute_path: name
+        # use_pkce: true # not sure if Authentik supports it
+      auth.generic_oauth.group_mapping:
+        role_attribute_path: |
+          contains(groups[*], 'Role-Grafana-Admin') && 'Admin' || contains(groups[*], 'Role-Grafana-Viewer') && 'Viewer'
+        org_id: 1
+      auth.basic:
+        enabled: false
+        disable_login_form: true
+      auth.anonymous:
+        enabled: false
+      grafana_net:
+        url: "https://grafana.net"
+    dashboardProviders:
+      dashboardproviders.yaml:
+        apiVersion: 1
+        providers:
+          - name: default
+            orgId: 1
+            folder: ""
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/default
+          - name: flux
+            orgId: 1
+            folder: Flux
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/flux
+          - name: kubernetes
+            orgId: 1
+            folder: Kubernetes
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/kubernetes
+          - name: nginx
+            orgId: 1
+            folder: Nginx
+            type: file
+            disableDeletion: false
+            editable: true
+            options:
+              path: /var/lib/grafana/dashboards/nginx
+    datasources:
+      datasources.yaml:
+        apiVersion: 1
+        deleteDatasources:
+          - { name: Prometheus, orgId: 1 }
+        datasources:
+          - name: Prometheus
+            type: prometheus
+            uid: prometheus
+            access: proxy
+            url: http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090
+            jsonData:
+              prometheusType: Prometheus
+            isDefault: true
+    dashboards:
+      default:
+        cloudflared:
+          gnetId: 17457 # https://grafana.com/grafana/dashboards/17457?tab=revisions
+          revision: 6
+          datasource:
+            - { name: DS_PROMETHEUS, value: Prometheus }
+        external-dns:
+          gnetId: 15038 # https://grafana.com/grafana/dashboards/15038?tab=revisions
+          revision: 1
+          datasource: Prometheus
+        cert-manager:
+          url: https://raw.githubusercontent.com/monitoring-mixins/website/master/assets/cert-manager/dashboards/cert-manager.json
+          datasource: Prometheus
+        node-exporter-full:
+          gnetId: 1860 # https://grafana.com/grafana/dashboards/1860?tab=revisions
+          revision: 31
+          datasource: Prometheus
+      flux:
+        flux-cluster:
+          url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/cluster.json
+          datasource: Prometheus
+        flux-control-plane:
+          url: https://raw.githubusercontent.com/fluxcd/flux2/main/manifests/monitoring/monitoring-config/dashboards/control-plane.json
+          datasource: Prometheus
+      kubernetes:
+        kubernetes-api-server:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-api-server.json
+          datasource: Prometheus
+        kubernetes-coredns:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-coredns.json
+          datasource: Prometheus
+        kubernetes-global:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-global.json
+          datasource: Prometheus
+        kubernetes-namespaces:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-namespaces.json
+          datasource: Prometheus
+        kubernetes-nodes:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-nodes.json
+          datasource: Prometheus
+        kubernetes-pods:
+          url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-pods.json
+          datasource: Prometheus
+      nginx:
+        nginx:
+          url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json
+          datasource: Prometheus
+        nginx-request-handling-performance:
+          url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/request-handling-performance.json
+          datasource: Prometheus
+    sidecar:
+      dashboards:
+        enabled: true
+        searchNamespace: ALL
+        labelValue: ""
+        label: grafana_dashboard
+        folderAnnotation: grafana_folder
+        provider:
+          disableDelete: true
+          foldersFromFilesStructure: true
+      datasources:
+        enabled: true
+        searchNamespace: ALL
+        labelValue: ""
+    serviceMonitor:
+      enabled: true
+    ingress:
+      enabled: true
+      ingressClassName: "ingress-nginx"
+      hosts:
+        - &host "${APP_DNS_GRAFANA}"
+      tls:
+        - hosts:
+            - *host
+    persistence:
+      enabled: false
+    testFramework:
+      enabled: false
--- a/kube/deploy/core/monitoring/grafana/app/secrets.yaml
+++ b/kube/deploy/core/monitoring/grafana/app/secrets.yaml
@@ -0,0 +1,14 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "grafana-secrets"
+  namespace: "monitoring"
+type: Opaque
+stringData:
+  GF_AUTH_GENERIC_OAUTH_CLIENT_ID: ${SECRET_GRAFANA_OIDC_ID}
+  GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ${SECRET_GRAFANA_OIDC_SECRET}
+  GF_AUTH_GENERIC_OAUTH_AUTH_URL: ${SECRET_AUTHENTIK_OIDC_URL_AUTHZ}
+  GF_AUTH_GENERIC_OAUTH_TOKEN_URL: ${SECRET_AUTHENTIK_OIDC_URL_TOKEN}
+  GF_AUTH_GENERIC_OAUTH_API_URL: ${SECRET_AUTHENTIK_OIDC_URL_USERINFO}
+  GF_AUTH_SIGNOUT_REDIRECT_URL: ${SECRET_GRAFANA_OIDC_URL_SIGNOUT}
--- a/kube/deploy/core/monitoring/grafana/ks.yaml
+++ b/kube/deploy/core/monitoring/grafana/ks.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: 1-core-monitoring-grafana-app
+  namespace: flux-system
+spec:
+  path: ./kube/deploy/core/monitoring/grafana/app
+  dependsOn:
+    - name: 1-core-monitoring-deps
--- a/kube/deploy/core/monitoring/grafana/kustomization.yaml
+++ b/kube/deploy/core/monitoring/grafana/kustomization.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ks.yaml
--- a/kube/deploy/core/monitoring/kps/app/config/kube-state-metrics.yaml
+++ b/kube/deploy/core/monitoring/kps/app/config/kube-state-metrics.yaml
@@ -0,0 +1,17 @@
+---
+kubeStateMetrics:
+  enabled: true
+kube-state-metrics:
+  metricLabelsAllowlist:
+    - "deployments=[*]"
+    - "persistentvolumeclaims=[*]"
+    - "pods=[*]"
+  prometheus:
+    monitor:
+      enabled: true
+      relabelings:
+        - action: replace
+          sourceLabels: ["__meta_kubernetes_pod_node_name"]
+          regex: ^(.*)$
+          replacement: $1
+          targetLabel: kubernetes_node
--- a/kube/deploy/core/monitoring/kps/app/config/kube.yaml
+++ b/kube/deploy/core/monitoring/kps/app/config/kube.yaml
@@ -0,0 +1,20 @@
+---
+kubelet:
+  enabled: true
+kubeApiServer:
+  enabled: true
+kubeControllerManager:
+  enabled: true
+  endpoints: &cp ["${IP_ROUTER_VLAN_K8S_PREFIX}1", "${IP_ROUTER_VLAN_K8S_PREFIX}2", "${IP_ROUTER_VLAN_K8S_PREFIX}3"]
+kubeEtcd:
+  enabled: true
+  endpoints: *cp
+  service:
+    enabled: true
+    port: 2381
+    targetPort: 2381
+kubeScheduler:
+  enabled: true
+  endpoints: *cp
+kubeProxy:
+  enabled: false # Disabled due to eBPF
--- a/kube/deploy/core/monitoring/kps/app/config/kustomization.yaml
+++ b/kube/deploy/core/monitoring/kps/app/config/kustomization.yaml
@@ -0,0 +1,12 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+configMapGenerator:
+  - name: "kps-config"
+    namespace: monitoring
+    files:
+      - kube.yaml
+      - prom.yaml
+      - kube-state-metrics.yaml
+configurations:
+  - kustomizeconfig.yaml
--- a/kube/deploy/core/monitoring/kps/app/config/kustomizeconfig.yaml
+++ b/kube/deploy/core/monitoring/kps/app/config/kustomizeconfig.yaml
@@ -0,0 +1,7 @@
+---
+nameReference:
+- kind: ConfigMap
+  version: v1
+  fieldSpecs:
+  - path: spec/valuesFrom/name
+    kind: HelmRelease
--- a/kube/deploy/core/monitoring/kps/app/config/prom.yaml
+++ b/kube/deploy/core/monitoring/kps/app/config/prom.yaml
@@ -0,0 +1,39 @@
+---
+prometheus:
+  ingress:
+    enabled: true
+    ingressClassName: ingress-nginx
+    hosts: &hostprom ["${APP_DNS_PROMETHEUS}"]
+    tls: [hosts: *hostprom]
+    paths: ["/"]
+    pathType: Prefix
+  prometheusSpec:
+    replicas: 2
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false
+    scrapeConfigSelectorNilUsesHelmValues: false
+    enableAdminAPI: true
+    walCompression: true
+    retentionSize: 15GB
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          storageClassName: block
+          resources:
+            requests:
+              storage: 20Gi
+    resources:
+      requests:
+        cpu: 150m
+        memory: 2048M
+      limits:
+        memory: 8192M
+    topologySpreadConstraints:
+      - maxSkew: 1
+        topologyKey: "kubernetes.io/hostname"
+        whenUnsatisfiable: DoNotSchedule
+        labelSelector:
+          matchLabels:
+            app: prometheus
--- a/kube/deploy/core/monitoring/kps/app/hr.yaml
+++ b/kube/deploy/core/monitoring/kps/app/hr.yaml
@@ -0,0 +1,39 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: kps
+  namespace: monitoring
+spec:
+  timeout: 30m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: 51.10.0
+      sourceRef:
+        name: prometheus-community
+        kind: HelmRepository
+        namespace: flux-system
+  valuesFrom:
+    - &vf
+      name: kps-config
+      kind: ConfigMap
+      valuesKey: kube.yaml
+    - <<: *vf
+      valuesKey: prom.yaml
+    - <<: *vf
+      valuesKey: kube-state-metrics.yaml
+  values:
+    crds:
+      enabled: false
+    cleanPrometheusOperatorObjectNames: true
+    alertmanager:
+      enabled: false
+    grafana:
+      enabled: false
+      forceDeployDashboards: true
+      sidecar:
+        dashboards:
+          multicluster:
+            etcd:
+              enabled: true
--- a/kube/deploy/core/monitoring/kps/ks.yaml
+++ b/kube/deploy/core/monitoring/kps/ks.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: 1-core-monitoring-kps-app
+  namespace: flux-system
+spec:
+  path: ./kube/deploy/core/monitoring/kps/app
+  dependsOn:
+    - name: 1-core-monitoring-deps
--- a/kube/deploy/core/monitoring/kps/kustomization.yaml
+++ b/kube/deploy/core/monitoring/kps/kustomization.yaml
@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ks.yaml