feat: add Mosquitto MQTT

kube/deploy/core/db/emqx/app/hr.yaml

@@ -1,34 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: &app emqx-operator
-  namespace: emqx
-spec:
-  interval: 5m
-  chart:
-    spec:
-      chart: emqx-operator
-      version: 2.2.28
-      sourceRef:
-        name: emqx
-        kind: HelmRepository
-        namespace: flux-system
-  dependsOn:
-    - name: cert-manager
-      namespace: cert-manager
-  values:
-    fullnameOverride: *app
-    singleNamespace: true
-    image:
-      repository: ghcr.io/emqx/emqx-operator
-  postRenderers:
-    - kustomize:
-        patches:
-          - target:
-              kind: Deployment
-              labelSelector: control-plane=controller-manager
-            patch: |
-              - op: add
-                path: "/spec/template/metadata/labels/egress.home.arpa~1apiserver"
-                value: allow

kube/deploy/core/db/emqx/app/netpol.yaml

@@ -1,32 +0,0 @@
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: &app emqx-operator-to-replicas
-  namespace: *app
-spec:
-  endpointSelector: &cluster
-    matchLabels:
-      app.kubernetes.io/name: emqx-operator
-  egress:
-    # same cluster
-    - toEndpoints:
-        - matchLabels:
-            apps.emqx.io/managed-by: emqx-operator
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: &app emqx-replicas-from-operator
-  namespace: *app
-spec:
-  endpointSelector: &cluster
-    matchLabels:
-      apps.emqx.io/managed-by: emqx-operator
-  egress:
-    # same cluster
-    - toEndpoints:
-        - matchLabels:
-            app.kubernetes.io/name: emqx-operator

kube/deploy/core/db/emqx/cluster/emqx.yaml

@@ -1,86 +0,0 @@
----
-# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/apps.emqx.io/emqx_v2beta1.json
-apiVersion: apps.emqx.io/v2beta1
-kind: EMQX
-metadata:
-  name: &app emqx
-  namespace: emqx
-spec:
-  image: public.ecr.aws/emqx/emqx:5.8.7@sha256:556aea6d62134524ecd1fcca53380b460b52995344dce571d484f042d9b15e7d
-  config:
-    # the `$${env}` in the env var means Flux's Kustomize controller should escape the envsubst and output `${env}` exactly
-    data: |
-      authentication {
-        enable = true
-        backend = "built_in_database"
-        mechanism = "password_based"
-        password_hash_algorithm {
-            name = "bcrypt"
-        }
-        user_id_type = "username"
-        bootstrap_file = "/secrets/init-user.json"
-        bootstrap_type = "plain"
-      }
-      authorization {
-        no_match = "deny"
-        deny_action = "ignore"
-        sources = [
-          {
-            enable = true
-            type = file
-            path = "/secrets/acl.conf"
-          # },
-          # {
-          #   enable = true
-          #   type = built_in_database
-          }
-        ]
-      }
-      listeners.ssl.default {
-        bind = "0.0.0.0:8883"
-        ssl_options {
-          certfile = "/tls/cert.pem"
-          keyfile = "/tls/key.pem"
-          gc_after_handshake = true
-          # no mTLS
-          verify = verify_none
-          fail_if_no_peer_cert = false
-        }
-      }
-  coreTemplate:
-    spec:
-      replicas: 2
-      envFrom: &envFrom
-        - secretRef:
-            name: emqx-secrets
-      extraVolumes:
-        - name: emqx-secrets
-          secret:
-            secretName: emqx-secrets
-        - name: tls
-          secret:
-            secretName: short-domain-tls
-      extraVolumeMounts:
-        - name: emqx-secrets
-          subPath: init-user.json
-          mountPath: /secrets/init-user.json
-          readOnly: true
-        - name: emqx-secrets
-          subPath: acl.conf
-          mountPath: /secrets/acl.conf
-          readOnly: true
-        - name: tls
-          subPath: tls.crt
-          mountPath: /tls/cert.pem
-          readOnly: true
-        - name: tls
-          subPath: tls.key
-          mountPath: /tls/key.pem
-          readOnly: true
-  listenersServiceTemplate:
-    metadata:
-      annotations:
-        coredns.io/hostname: "${APP_DNS_EMQX}" # TLS SNI
-        io.cilium/lb-ipam-ips: "${APP_IP_EMQX}"
-    spec:
-      type: LoadBalancer

kube/deploy/core/db/emqx/cluster/es.yaml

@@ -1,48 +0,0 @@
----
-# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: &name emqx-secrets
-  namespace: emqx
-spec:
-  refreshInterval: 1m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: 1p
-  dataFrom:
-    - extract:
-        key: "EMQX - ${CLUSTER_NAME}"
-  target:
-    creationPolicy: Owner
-    deletionPolicy: Retain
-    name: *name
-    template:
-      type: Opaque
-      data:
-        EMQX_DASHBOARD__DEFAULT_USERNAME: "{{ .EMQX_DASHBOARD__DEFAULT_USERNAME }}"
-        EMQX_DASHBOARD__DEFAULT_PASSWORD: "{{ .EMQX_DASHBOARD__DEFAULT_PASSWORD }}"
-        X_EMQX_APIKEY_KEY: "{{ .X_EMQX_APIKEY_KEY }}"
-        X_EMQX_APIKEY_SECRET: "{{ .X_EMQX_APIKEY_SECRET }}"
-        init-user.json: |
-          [
-            {"user_id": "{{ .X_EMQX_MQTT_Z2M_USERNAME }}", "password": "{{ .X_EMQX_MQTT_Z2M_PASSWORD }}", "is_superuser": false},
-            {"user_id": "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}", "password": "{{ .X_EMQX_MQTT_VALETUDO_PASSWORD }}", "is_superuser": false},
-            {"user_id": "{{ .X_EMQX_MQTT_HASS_USERNAME }}", "password": "{{ .X_EMQX_MQTT_HASS_PASSWORD }}", "is_superuser": false}
-          ]
-        acl.conf: |
-          %% Home Assistant
-          {allow, {user, "{{ .X_EMQX_MQTT_HASS_USERNAME }}"}, all, ["homeassistant/#", "hass/#", "zigbee2mqtt/#", "valetudo/#"]}.
-
-          %% Zigbee2MQTT, data and HASS discovery
-          %%{allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, publish, ["homeassistant/#"]}.
-          %%{allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, subscribe, ["homeassistant/status"]}.
-          {allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, all, ["homeassistant/#", "hass/#", "zigbee2mqtt/#"]}.
-
-          %% Valetudo, data and HASS discovery
-          {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, publish, ["homeassistant/#"]}.
-          {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, subscribe, ["homeassistant/status"]}.
-          {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, all, ["valetudo/#"]}.
-
-          %% Default Deny All
-          %%{deny, all}. %% commented out to allow failures to be logged

kube/deploy/core/db/emqx/ks.yaml

@@ -1,29 +0,0 @@
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: emqx-app
-  namespace: flux-system
-  labels: &l
-    app.kubernetes.io/name: "emqx"
-spec:
-  commonMetadata:
-    labels: *l
-  path: ./kube/deploy/core/db/emqx/app
-  targetNamespace: "emqx"
-  dependsOn: []
----
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: emqx-cluster
-  namespace: flux-system
-  labels: &l
-    app.kubernetes.io/name: "emqx"
-spec:
-  commonMetadata:
-    labels: *l
-  path: ./kube/deploy/core/db/emqx/cluster
-  targetNamespace: "emqx"
-  dependsOn:
-    - name: emqx-app

kube/deploy/core/db/mosquitto/app/es.yaml

@@ -0,0 +1,65 @@
+---
+# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: &name mosquitto-secrets
+  namespace: mosquitto
+spec:
+  refreshInterval: 1m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: 1p
+  dataFrom:
+    - extract:
+        key: "Mosquitto - ${CLUSTER_NAME}"
+  target:
+    creationPolicy: Owner
+    deletionPolicy: Retain
+    name: *name
+    template:
+      type: Opaque
+      data:
+        passwd: |
+          {{ .X_EMQX_MQTT_BRIDGE_USERNAME }}:{{ .X_EMQX_MQTT_BRIDGE_PASSWORD }}
+          {{ .X_EMQX_MQTT_HASS_USERNAME }}:{{ .X_EMQX_MQTT_HASS_PASSWORD }}
+          {{ .X_EMQX_MQTT_Z2M_USERNAME }}:{{ .X_EMQX_MQTT_Z2M_PASSWORD }}
+          {{ .X_EMQX_MQTT_VALETUDO_USERNAME }}:{{ .X_EMQX_MQTT_VALETUDO_PASSWORD }}
+          {{ .X_EMQX_MQTT_ESPHOME_CN105_USERNAME }}:{{ .X_EMQX_MQTT_ESPHOME_CN105_PASSWORD }}
+        acl.conf: |
+          user {{ .X_EMQX_MQTT_BRIDGE_USERNAME }}
+          topic readwrite #
+
+          user {{ .X_EMQX_MQTT_HASS_USERNAME }}
+          topic readwrite homeassistant/#
+          topic readwrite hass/#
+          topic readwrite zigbee2mqtt/#
+          topic readwrite valetudo/#
+
+          user {{ .X_EMQX_MQTT_Z2M_USERNAME }}
+          topic write homeassistant/#
+          topic read homeassistant/status
+          topic readwrite zigbee2mqtt/#
+
+          user {{ .X_EMQX_MQTT_VALETUDO_USERNAME }}
+          topic write homeassistant/#
+          topic read homeassistant/status
+          topic readwrite valetudo/#
+
+          user {{ .X_EMQX_MQTT_ESPHOME_CN105_USERNAME }}
+          topic write homeassistant/#
+          topic read homeassistant/status
+          topic readwrite esphome/#
+          topic readwrite cn105/+/#
+        bridge-1.conf: |
+          connection mosquitto-0-bridge
+          address mosquitto-0.mosquitto-bridge.mosquitto.svc.cluster.local:2883
+          topic # both 0
+          local_username {{ .X_EMQX_MQTT_BRIDGE_USERNAME }}
+          local_password {{ .X_EMQX_MQTT_BRIDGE_PASSWORD }}
+          local_clientid mosquitto-0-bridge
+          remote_username {{ .X_EMQX_MQTT_BRIDGE_USERNAME }}
+          remote_password {{ .X_EMQX_MQTT_BRIDGE_PASSWORD }}
+          remote_clientid mosquitto-1-bridge
+          try_private true
+          bridge_protocol_version mqttv50

kube/deploy/core/db/mosquitto/app/hr.yaml

@@ -0,0 +1,168 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/app-template-4.1.2/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: &app mosquitto
+  namespace: *app
+spec:
+  interval: 5m
+  chart:
+    spec:
+      chart: app-template
+      version: 4.1.2
+      sourceRef:
+        name: bjw-s
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    controllers:
+      app:
+        type: statefulset
+        replicas: 2
+        containers:
+          app:
+            image: &img
+              repository: public.ecr.aws/docker/library/eclipse-mosquitto
+              tag: 2.0.22@sha256:d219d3a72847f3aed6a1d66975972d3b17f86e39e8f6f6b86b4088b879c1a2d6
+            command: ["sh", "-c"]
+            args:
+              - exec mosquitto -c "/config/$(hostname).conf"
+            env: &env
+              TZ: "${CONFIG_TZ}"
+            securityContext: &sc
+              readOnlyRootFilesystem: true
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop: ["ALL"]
+            resources:
+              requests:
+                cpu: 10m
+                memory: 16Mi
+              limits:
+                cpu: "1"
+                memory: 256Mi
+            probes:
+              liveness:
+                enabled: true
+                custom: true
+                spec:
+                  exec:
+                    command:
+                      - /bin/sh
+                      - -c
+                      - >-
+                        /usr/bin/mosquitto_pub -h localhost -p 1883 -t mosquitto/healthcheck/$(hostname)/liveness -m "ok $(date)" -r -q 0
+              readiness:
+                enabled: true
+    service:
+      app:
+        controller: app
+        type: LoadBalancer
+        annotations:
+          coredns.io/hostname: "${APP_DNS_MOSQUITTO:=mosquitto}"
+          lbipam.cilium.io/ips: "${APP_IP_MOSQUITTO:=127.0.0.1}"
+        ports:
+          # mqtt:
+          #   port: 1883
+          #   protocol: TCP
+          #   appProtocol: mqtt
+          tls:
+            port: 8883
+            protocol: TCP
+            appProtocol: secure-mqtt
+      bridge:
+        primary: false
+        controller: app
+        ports:
+          http:
+            port: 2883
+            protocol: TCP
+            appProtocol: mqtt
+    persistence:
+      config:
+        type: configMap
+        identifier: config
+        globalMounts:
+          - path: /config
+      secrets:
+        type: secret
+        name: mosquitto-secrets
+        defaultMode: 0400
+        globalMounts:
+          - path: /secrets
+      tls:
+        type: secret
+        name: short-domain-tls
+        defaultMode: 0400
+        globalMounts:
+          - path: /tls
+    configMaps:
+      config:
+        data:
+          common.conf: |
+            log_type error
+            log_type warning
+
+            allow_anonymous false
+            per_listener_settings false
+            password_file /secrets/passwd
+            acl_file /secrets/acl.conf
+
+            listener 1883
+            protocol mqtt
+            use_username_as_clientid true
+
+            listener 8883
+            protocol mqtt
+            use_username_as_clientid true
+            certfile /tls/tls.crt
+            keyfile /tls/tls.key
+
+            listener 2883
+            protocol mqtt
+            use_username_as_clientid false
+          mosquitto-0.conf: |
+            include_dir /config/common
+          mosquitto-1.conf: |
+            include_dir /config/common
+            include_dir /secrets/bridge
+    defaultPodOptions:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      hostAliases:
+        - ip: "${APP_IP_AUTHENTIK:=127.0.0.1}"
+          hostnames: ["${APP_DNS_AUTHENTIK:=authentik}"]
+      dnsConfig:
+        options:
+          - name: ndots
+            value: "1"
+      hostUsers: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: &uid 1000
+        runAsGroup: *uid
+        fsGroup: *uid
+        fsGroupChangePolicy: Always
+        seccompProfile: { type: "RuntimeDefault" }
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: "{{ .Release.Name }}"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: "fuckoff.home.arpa/{{ .Release.Name }}"
+                    operator: DoesNotExist
+    networkpolicies:
+      same-ns:
+        controller: app
+        policyTypes: [Ingress, Egress]
+        rules:
+          ingress: [from: [{podSelector: {}}]]
+          egress: [to: [{podSelector: {}}]]

kube/deploy/core/db/mosquitto/app/netpol.yaml

@@ -3,59 +3,51 @@
 apiVersion: cilium.io/v2
 kind: CiliumClusterwideNetworkPolicy
 metadata:
-  name: "apps-to-emqx"
+  name: "apps-to-mosquitto"
 spec:
   endpointSelector:
     matchLabels:
-      db.home.arpa/emqx: "emqx"
+      db.home.arpa/mqtt: "allow"
   egress:
-    - toEndpoints: &emqx
+    - toEndpoints: &mosquitto
         - matchLabels:
-            io.kubernetes.pod.namespace: emqx
-            apps.emqx.io/instance: emqx
+            io.kubernetes.pod.namespace: mosquitto
+            app.kubernetes.io/name: mosquitto
       toPorts:
         - ports:
             - port: "8883"
               protocol: TCP
-    - toEndpoints: *emqx
+    - toEndpoints: *mosquitto
       icmps: [{}]
 ---
 # yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
-  name: &app emqx
+  name: &app mosquitto
   namespace: *app
 spec:
   endpointSelector: &cluster
     matchLabels:
-      apps.emqx.io/instance: emqx
+      app.kubernetes.io/name: mosquitto
   ingress:
-    # EMQX operator (readiness gate)
-    - fromEndpoints:
-        - matchLabels:
-            app.kubernetes.io/name: emqx-operator
-    # same EMQX cluster
+    # same mosquitto cluster
     - fromEndpoints: [*cluster]
     # labelled pods
     - fromEndpoints: &labelled
         - matchExpressions:
             - key: io.kubernetes.pod.namespace
               operator: Exists
-            - key: db.home.arpa/emqx
+            - key: db.home.arpa/mqtt
               operator: In
-              values: [emqx]
+              values: [allow]
       toPorts: &ports
         - ports:
             - port: "8883"
               protocol: TCP
     - fromEndpoints: *labelled
       icmps: [{}]
-    # operator
-    - fromEndpoints:
-        - matchLabels:
-            io.kubernetes.pod.namespace: emqx
-            app.kubernetes.io/name: emqx-operator
+    # Valetudo
     - fromCIDRSet:
         - cidr: "${IP_ROBOROCK}/32"
       toPorts: *ports

kube/deploy/core/db/mosquitto/ks.yaml

@@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: mosquitto-app
+  namespace: flux-system
+  labels: &l
+    app.kubernetes.io/name: "mosquitto"
+spec:
+  targetNamespace: "mosquitto"
+  commonMetadata:
+    labels: *l
+  path: ./kube/deploy/core/db/mosquitto/app
+  components:
+    - ../../../../core/flux-system/alerts/template/
+  dependsOn:
+    - name: crds
+      namespace: flux-system

kube/deploy/core/db/mosquitto/ns.yaml

@@ -2,9 +2,9 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: emqx
+  name: mosquitto
   labels:
     kustomize.toolkit.fluxcd.io/prune: disabled
-    pod-security.kubernetes.io/enforce: &ps baseline # operator securityContext is meh
+    pod-security.kubernetes.io/enforce: &ps restricted
     pod-security.kubernetes.io/audit: *ps
     pod-security.kubernetes.io/warn: *ps