diff --git a/kube/clusters/biohazard/flux/kustomization.yaml b/kube/clusters/biohazard/flux/kustomization.yaml index 339fec55..e3dbef77 100644 --- a/kube/clusters/biohazard/flux/kustomization.yaml +++ b/kube/clusters/biohazard/flux/kustomization.yaml @@ -52,6 +52,7 @@ resources: - ../../../deploy/core/monitoring/grafana/ - ../../../deploy/core/monitoring/alertmanager/ - ../../../deploy/core/monitoring/karma/ + - ../../../deploy/core/monitoring/fluentbit/ - ../../../deploy/core/monitoring/node-exporter/ - ../../../deploy/core/monitoring/smartctl-exporter/ - ../../../deploy/core/monitoring/intel-gpu-exporter/ diff --git a/kube/deploy/core/monitoring/fluentbit/app/config/fluent-bit.yaml b/kube/deploy/core/monitoring/fluentbit/app/config/fluent-bit.yaml new file mode 100644 index 00000000..9cb7afd0 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/app/config/fluent-bit.yaml @@ -0,0 +1,44 @@ +--- +pipeline: + inputs: + - name: tail # https://docs.fluentbit.io/manual/pipeline/inputs/tail + alias: kube + path: /var/log/containers/*.log + exclude_path: /var/log/*fluent* + multiline.parser: cri + tag: kube.* + filters: + # enrich and process logs with k8s metadata + - name: kubernetes # https://docs.fluentbit.io/manual/pipeline/filters/kubernetes + alias: kube + match: kube.* + buffer_size: 100MB + merge_log: on + "k8s-logging.parser": on + "k8s-logging.exclude": on + use_kubelet: on + kubelet_host: "${KUBE_NODE_IP}" + # Lift out the kubernetes labels so we can rename them + - name: nest + match: kube.* + operation: lift + nested_under: kubernetes + add_prefix: kube_ + - name: nest + match: kube.* + operation: lift + nested_under: k_labels + add_prefix: k_labels_ + outputs: + # Victoria Logs + - name: http + match: '*' + host: victoria-logs.monitoring.svc + port: 9428 + uri: '/insert/jsonline?_stream_fields=stream,kube_namespace_name,kube_pod_name,kube_container_name&_msg_field=log&_time_field=date' + compress: gzip + format: json_lines + json_date_format: iso8601 + header: + - 'AccountID 0' + - 'ProjectID 0' diff --git a/kube/deploy/core/monitoring/fluentbit/app/config/kustomization.yaml b/kube/deploy/core/monitoring/fluentbit/app/config/kustomization.yaml new file mode 100644 index 00000000..00202658 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/app/config/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +configMapGenerator: + - name: fluentbit-config + namespace: fluentbit + files: + - fluent-bit.yaml +generatorOptions: + disableNameSuffixHash: true diff --git a/kube/deploy/core/monitoring/fluentbit/app/hr.yaml b/kube/deploy/core/monitoring/fluentbit/app/hr.yaml new file mode 100644 index 00000000..dee912c5 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/app/hr.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: &app fluentbit + namespace: *app +spec: + interval: 5m + chart: + spec: + chart: app-template + version: 3.1.0 + sourceRef: + name: bjw-s + kind: HelmRepository + namespace: flux-system + values: + controllers: + fluentbit: + type: daemonset + containers: + main: + image: &img + repository: cr.fluentbit.io/fluent/fluent-bit + tag: 3.1.6@sha256:def31c22bfc23002354d0cc2beb177e4da1d87c68f49e8fee76c896e8c6fdd44 + args: ["--config=/config/fluent-bit.yaml"] + env: &env + TZ: "${CONFIG_TZ}" + KUBE_NODE_IP: + valueFrom: + fieldRef: + fieldPath: status.hostIP + securityContext: &sc + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: "10m" + memory: "128Mi" + limits: + cpu: "3000m" + memory: "6Gi" + #probes: # TODO: HTTP healthchecks via health API + # liveness: + # enabled: true + # readiness: + # enabled: true + # #startup: + # # enabled: true + #service: + # fluentbit: + # controller: fluentbit + # ports: + # http: + # port: 8080 + # protocol: HTTP + # appProtocol: http + persistence: + config: + type: configMap + name: fluentbit-config + globalMounts: + - path: /config + logs-containers: # normal Talos disk mount + type: hostPath + hostPath: &path "/var/log/containers" + globalMounts: + - path: *path + logs-pods: # jank tmpfs bind mount from /run on Talos + type: hostPath + hostPath: &path "/var/log/pods" + globalMounts: + - path: *path + defaultPodOptions: + automountServiceAccountToken: false + enableServiceLinks: false + securityContext: + runAsNonRoot: true + runAsUser: &uid ${APP_UID_FLUENTBIT:=1000} + runAsGroup: *uid + fsGroup: *uid + fsGroupChangePolicy: Always + seccompProfile: { type: "RuntimeDefault" } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: *app + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: fuckoff.home.arpa/fluentbit + operator: DoesNotExist + networkpolicies: + same-ns: + controller: fluentbit + policyTypes: [Ingress, Egress] + rules: + egress: + - to: + - podSelector: + matchLabels: + app.kubernetes.io/name: victoria-logs + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring diff --git a/kube/deploy/core/monitoring/fluentbit/ks.yaml b/kube/deploy/core/monitoring/fluentbit/ks.yaml new file mode 100644 index 00000000..5fe363a9 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/ks.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: fluentbit-app + namespace: flux-system + labels: &l + app.kubernetes.io/name: "fluentbit" +spec: + commonMetadata: + labels: *l + path: ./kube/deploy/core/monitoring/fluentbit/app + targetNamespace: "fluentbit" + dependsOn: [] diff --git a/kube/deploy/core/monitoring/fluentbit/kustomization.yaml b/kube/deploy/core/monitoring/fluentbit/kustomization.yaml new file mode 100644 index 00000000..5eeb2657 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ns.yaml + - ks.yaml diff --git a/kube/deploy/core/monitoring/fluentbit/ns.yaml b/kube/deploy/core/monitoring/fluentbit/ns.yaml new file mode 100644 index 00000000..ba3ac919 --- /dev/null +++ b/kube/deploy/core/monitoring/fluentbit/ns.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: fluentbit + labels: + kustomize.toolkit.fluxcd.io/prune: disabled + pod-security.kubernetes.io/enforce: &ps privileged # hostPath + pod-security.kubernetes.io/audit: *ps + pod-security.kubernetes.io/warn: *ps