feat: add fluentbit

kube/clusters/biohazard/flux/kustomization.yaml

@@ -52,6 +52,7 @@ resources:
   - ../../../deploy/core/monitoring/grafana/
   - ../../../deploy/core/monitoring/alertmanager/
   - ../../../deploy/core/monitoring/karma/
+  - ../../../deploy/core/monitoring/fluentbit/
   - ../../../deploy/core/monitoring/node-exporter/
   - ../../../deploy/core/monitoring/smartctl-exporter/
   - ../../../deploy/core/monitoring/intel-gpu-exporter/

kube/deploy/core/monitoring/fluentbit/app/config/fluent-bit.yaml

@@ -0,0 +1,44 @@
+---
+pipeline:
+  inputs:
+    - name: tail # https://docs.fluentbit.io/manual/pipeline/inputs/tail
+      alias: kube
+      path: /var/log/containers/*.log
+      exclude_path: /var/log/*fluent*
+      multiline.parser: cri
+      tag: kube.*
+  filters:
+    # enrich and process logs with k8s metadata
+    - name: kubernetes # https://docs.fluentbit.io/manual/pipeline/filters/kubernetes
+      alias: kube
+      match: kube.*
+      buffer_size: 100MB
+      merge_log: on
+      "k8s-logging.parser": on
+      "k8s-logging.exclude": on
+      use_kubelet: on
+      kubelet_host: "${KUBE_NODE_IP}"
+    # Lift out the kubernetes labels so we can rename them
+    - name: nest
+      match: kube.*
+      operation: lift
+      nested_under: kubernetes
+      add_prefix: kube_
+    - name: nest
+      match: kube.*
+      operation: lift
+      nested_under: k_labels
+      add_prefix: k_labels_
+  outputs:
+    # Victoria Logs
+    - name: http
+      match: '*'
+      host: victoria-logs.monitoring.svc
+      port: 9428
+      uri: '/insert/jsonline?_stream_fields=stream,kube_namespace_name,kube_pod_name,kube_container_name&_msg_field=log&_time_field=date'
+      compress: gzip
+      format: json_lines
+      json_date_format: iso8601
+      header:
+        - 'AccountID 0'
+        - 'ProjectID 0'

kube/deploy/core/monitoring/fluentbit/app/config/kustomization.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+configMapGenerator:
+  - name: fluentbit-config
+    namespace: fluentbit
+    files:
+      - fluent-bit.yaml
+generatorOptions:
+  disableNameSuffixHash: true

kube/deploy/core/monitoring/fluentbit/app/hr.yaml

@@ -0,0 +1,112 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: &app fluentbit
+  namespace: *app
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.1.0
+      sourceRef:
+        name: bjw-s
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    controllers:
+      fluentbit:
+        type: daemonset
+        containers:
+          main:
+            image: &img
+              repository: cr.fluentbit.io/fluent/fluent-bit
+              tag: 3.1.6@sha256:def31c22bfc23002354d0cc2beb177e4da1d87c68f49e8fee76c896e8c6fdd44
+            args: ["--config=/config/fluent-bit.yaml"]
+            env: &env
+              TZ: "${CONFIG_TZ}"
+              KUBE_NODE_IP:
+                valueFrom:
+                  fieldRef:
+                    fieldPath: status.hostIP
+            securityContext: &sc
+              readOnlyRootFilesystem: true
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop: ["ALL"]
+            resources:
+              requests:
+                cpu: "10m"
+                memory: "128Mi"
+              limits:
+                cpu: "3000m"
+                memory: "6Gi"
+            #probes: # TODO: HTTP healthchecks via health API
+            #  liveness:
+            #    enabled: true
+            #  readiness:
+            #    enabled: true
+            #  #startup:
+            #  #  enabled: true
+    #service:
+    #  fluentbit:
+    #    controller: fluentbit
+    #    ports:
+    #      http:
+    #        port: 8080
+    #        protocol: HTTP
+    #        appProtocol: http
+    persistence:
+      config:
+        type: configMap
+        name: fluentbit-config
+        globalMounts:
+          - path: /config
+      logs-containers: # normal Talos disk mount
+        type: hostPath
+        hostPath: &path "/var/log/containers"
+        globalMounts:
+          - path: *path
+      logs-pods: # jank tmpfs bind mount from /run on Talos
+        type: hostPath
+        hostPath: &path "/var/log/pods"
+        globalMounts:
+          - path: *path
+    defaultPodOptions:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: &uid ${APP_UID_FLUENTBIT:=1000}
+        runAsGroup: *uid
+        fsGroup: *uid
+        fsGroupChangePolicy: Always
+        seccompProfile: { type: "RuntimeDefault" }
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: *app
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: fuckoff.home.arpa/fluentbit
+                    operator: DoesNotExist
+    networkpolicies:
+      same-ns:
+        controller: fluentbit
+        policyTypes: [Ingress, Egress]
+        rules:
+          egress:
+            - to:
+                - podSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: victoria-logs
+                  namespaceSelector:
+                    matchLabels:
+                      kubernetes.io/metadata.name: monitoring

kube/deploy/core/monitoring/fluentbit/ks.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: fluentbit-app
+  namespace: flux-system
+  labels: &l
+    app.kubernetes.io/name: "fluentbit"
+spec:
+  commonMetadata:
+    labels: *l
+  path: ./kube/deploy/core/monitoring/fluentbit/app
+  targetNamespace: "fluentbit"
+  dependsOn: []

kube/deploy/core/monitoring/fluentbit/kustomization.yaml

@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ns.yaml
+  - ks.yaml

kube/deploy/core/monitoring/fluentbit/ns.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: fluentbit
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+    pod-security.kubernetes.io/enforce: &ps privileged # hostPath
+    pod-security.kubernetes.io/audit: *ps
+    pod-security.kubernetes.io/warn: *ps