From f2b96f6a3246d653fbcf6b23025525fa0b3cbe59 Mon Sep 17 00:00:00 2001 From: JJGadgets Date: Tue, 29 Jul 2025 19:33:42 +0800 Subject: [PATCH] feat: add Mosquitto MQTT --- kube/deploy/core/db/emqx/app/hr.yaml | 34 ---- kube/deploy/core/db/emqx/app/netpol.yaml | 32 ---- kube/deploy/core/db/emqx/cluster/emqx.yaml | 86 --------- kube/deploy/core/db/emqx/cluster/es.yaml | 48 ----- kube/deploy/core/db/emqx/ks.yaml | 29 --- kube/deploy/core/db/mosquitto/app/es.yaml | 65 +++++++ kube/deploy/core/db/mosquitto/app/hr.yaml | 168 ++++++++++++++++++ .../cluster => mosquitto/app}/netpol.yaml | 32 ++-- kube/deploy/core/db/mosquitto/ks.yaml | 18 ++ .../db/{emqx => mosquitto}/kustomization.yaml | 0 .../core/db/{emqx => mosquitto}/ns.yaml | 4 +- 11 files changed, 265 insertions(+), 251 deletions(-) delete mode 100644 kube/deploy/core/db/emqx/app/hr.yaml delete mode 100644 kube/deploy/core/db/emqx/app/netpol.yaml delete mode 100644 kube/deploy/core/db/emqx/cluster/emqx.yaml delete mode 100644 kube/deploy/core/db/emqx/cluster/es.yaml delete mode 100644 kube/deploy/core/db/emqx/ks.yaml create mode 100644 kube/deploy/core/db/mosquitto/app/es.yaml create mode 100644 kube/deploy/core/db/mosquitto/app/hr.yaml rename kube/deploy/core/db/{emqx/cluster => mosquitto/app}/netpol.yaml (65%) create mode 100644 kube/deploy/core/db/mosquitto/ks.yaml rename kube/deploy/core/db/{emqx => mosquitto}/kustomization.yaml (100%) rename kube/deploy/core/db/{emqx => mosquitto}/ns.yaml (65%) diff --git a/kube/deploy/core/db/emqx/app/hr.yaml b/kube/deploy/core/db/emqx/app/hr.yaml deleted file mode 100644 index 8c198446..00000000 --- a/kube/deploy/core/db/emqx/app/hr.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -apiVersion: helm.toolkit.fluxcd.io/v2 -kind: HelmRelease -metadata: - name: &app emqx-operator - namespace: emqx -spec: - interval: 5m - chart: - spec: - chart: emqx-operator - version: 2.2.28 - sourceRef: - name: emqx - kind: HelmRepository - namespace: flux-system - dependsOn: - - name: cert-manager - namespace: cert-manager - values: - fullnameOverride: *app - singleNamespace: true - image: - repository: ghcr.io/emqx/emqx-operator - postRenderers: - - kustomize: - patches: - - target: - kind: Deployment - labelSelector: control-plane=controller-manager - patch: | - - op: add - path: "/spec/template/metadata/labels/egress.home.arpa~1apiserver" - value: allow diff --git a/kube/deploy/core/db/emqx/app/netpol.yaml b/kube/deploy/core/db/emqx/app/netpol.yaml deleted file mode 100644 index 8fc87ae7..00000000 --- a/kube/deploy/core/db/emqx/app/netpol.yaml +++ /dev/null @@ -1,32 +0,0 @@ ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: &app emqx-operator-to-replicas - namespace: *app -spec: - endpointSelector: &cluster - matchLabels: - app.kubernetes.io/name: emqx-operator - egress: - # same cluster - - toEndpoints: - - matchLabels: - apps.emqx.io/managed-by: emqx-operator ---- -# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json -apiVersion: cilium.io/v2 -kind: CiliumNetworkPolicy -metadata: - name: &app emqx-replicas-from-operator - namespace: *app -spec: - endpointSelector: &cluster - matchLabels: - apps.emqx.io/managed-by: emqx-operator - egress: - # same cluster - - toEndpoints: - - matchLabels: - app.kubernetes.io/name: emqx-operator diff --git a/kube/deploy/core/db/emqx/cluster/emqx.yaml b/kube/deploy/core/db/emqx/cluster/emqx.yaml deleted file mode 100644 index 1146f0bd..00000000 --- a/kube/deploy/core/db/emqx/cluster/emqx.yaml +++ /dev/null @@ -1,86 +0,0 @@ ---- -# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/apps.emqx.io/emqx_v2beta1.json -apiVersion: apps.emqx.io/v2beta1 -kind: EMQX -metadata: - name: &app emqx - namespace: emqx -spec: - image: public.ecr.aws/emqx/emqx:5.8.7@sha256:556aea6d62134524ecd1fcca53380b460b52995344dce571d484f042d9b15e7d - config: - # the `$${env}` in the env var means Flux's Kustomize controller should escape the envsubst and output `${env}` exactly - data: | - authentication { - enable = true - backend = "built_in_database" - mechanism = "password_based" - password_hash_algorithm { - name = "bcrypt" - } - user_id_type = "username" - bootstrap_file = "/secrets/init-user.json" - bootstrap_type = "plain" - } - authorization { - no_match = "deny" - deny_action = "ignore" - sources = [ - { - enable = true - type = file - path = "/secrets/acl.conf" - # }, - # { - # enable = true - # type = built_in_database - } - ] - } - listeners.ssl.default { - bind = "0.0.0.0:8883" - ssl_options { - certfile = "/tls/cert.pem" - keyfile = "/tls/key.pem" - gc_after_handshake = true - # no mTLS - verify = verify_none - fail_if_no_peer_cert = false - } - } - coreTemplate: - spec: - replicas: 2 - envFrom: &envFrom - - secretRef: - name: emqx-secrets - extraVolumes: - - name: emqx-secrets - secret: - secretName: emqx-secrets - - name: tls - secret: - secretName: short-domain-tls - extraVolumeMounts: - - name: emqx-secrets - subPath: init-user.json - mountPath: /secrets/init-user.json - readOnly: true - - name: emqx-secrets - subPath: acl.conf - mountPath: /secrets/acl.conf - readOnly: true - - name: tls - subPath: tls.crt - mountPath: /tls/cert.pem - readOnly: true - - name: tls - subPath: tls.key - mountPath: /tls/key.pem - readOnly: true - listenersServiceTemplate: - metadata: - annotations: - coredns.io/hostname: "${APP_DNS_EMQX}" # TLS SNI - io.cilium/lb-ipam-ips: "${APP_IP_EMQX}" - spec: - type: LoadBalancer diff --git a/kube/deploy/core/db/emqx/cluster/es.yaml b/kube/deploy/core/db/emqx/cluster/es.yaml deleted file mode 100644 index b62a4535..00000000 --- a/kube/deploy/core/db/emqx/cluster/es.yaml +++ /dev/null @@ -1,48 +0,0 @@ ---- -# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json -apiVersion: external-secrets.io/v1 -kind: ExternalSecret -metadata: - name: &name emqx-secrets - namespace: emqx -spec: - refreshInterval: 1m - secretStoreRef: - kind: ClusterSecretStore - name: 1p - dataFrom: - - extract: - key: "EMQX - ${CLUSTER_NAME}" - target: - creationPolicy: Owner - deletionPolicy: Retain - name: *name - template: - type: Opaque - data: - EMQX_DASHBOARD__DEFAULT_USERNAME: "{{ .EMQX_DASHBOARD__DEFAULT_USERNAME }}" - EMQX_DASHBOARD__DEFAULT_PASSWORD: "{{ .EMQX_DASHBOARD__DEFAULT_PASSWORD }}" - X_EMQX_APIKEY_KEY: "{{ .X_EMQX_APIKEY_KEY }}" - X_EMQX_APIKEY_SECRET: "{{ .X_EMQX_APIKEY_SECRET }}" - init-user.json: | - [ - {"user_id": "{{ .X_EMQX_MQTT_Z2M_USERNAME }}", "password": "{{ .X_EMQX_MQTT_Z2M_PASSWORD }}", "is_superuser": false}, - {"user_id": "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}", "password": "{{ .X_EMQX_MQTT_VALETUDO_PASSWORD }}", "is_superuser": false}, - {"user_id": "{{ .X_EMQX_MQTT_HASS_USERNAME }}", "password": "{{ .X_EMQX_MQTT_HASS_PASSWORD }}", "is_superuser": false} - ] - acl.conf: | - %% Home Assistant - {allow, {user, "{{ .X_EMQX_MQTT_HASS_USERNAME }}"}, all, ["homeassistant/#", "hass/#", "zigbee2mqtt/#", "valetudo/#"]}. - - %% Zigbee2MQTT, data and HASS discovery - %%{allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, publish, ["homeassistant/#"]}. - %%{allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, subscribe, ["homeassistant/status"]}. - {allow, {user, "{{ .X_EMQX_MQTT_Z2M_USERNAME }}"}, all, ["homeassistant/#", "hass/#", "zigbee2mqtt/#"]}. - - %% Valetudo, data and HASS discovery - {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, publish, ["homeassistant/#"]}. - {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, subscribe, ["homeassistant/status"]}. - {allow, {user, "{{ .X_EMQX_MQTT_VALETUDO_USERNAME }}"}, all, ["valetudo/#"]}. - - %% Default Deny All - %%{deny, all}. %% commented out to allow failures to be logged diff --git a/kube/deploy/core/db/emqx/ks.yaml b/kube/deploy/core/db/emqx/ks.yaml deleted file mode 100644 index 2b0a8c35..00000000 --- a/kube/deploy/core/db/emqx/ks.yaml +++ /dev/null @@ -1,29 +0,0 @@ ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: emqx-app - namespace: flux-system - labels: &l - app.kubernetes.io/name: "emqx" -spec: - commonMetadata: - labels: *l - path: ./kube/deploy/core/db/emqx/app - targetNamespace: "emqx" - dependsOn: [] ---- -apiVersion: kustomize.toolkit.fluxcd.io/v1 -kind: Kustomization -metadata: - name: emqx-cluster - namespace: flux-system - labels: &l - app.kubernetes.io/name: "emqx" -spec: - commonMetadata: - labels: *l - path: ./kube/deploy/core/db/emqx/cluster - targetNamespace: "emqx" - dependsOn: - - name: emqx-app diff --git a/kube/deploy/core/db/mosquitto/app/es.yaml b/kube/deploy/core/db/mosquitto/app/es.yaml new file mode 100644 index 00000000..78ce7fcc --- /dev/null +++ b/kube/deploy/core/db/mosquitto/app/es.yaml @@ -0,0 +1,65 @@ +--- +# yaml-language-server: $schema=https://crds.jank.ing/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: &name mosquitto-secrets + namespace: mosquitto +spec: + refreshInterval: 1m + secretStoreRef: + kind: ClusterSecretStore + name: 1p + dataFrom: + - extract: + key: "Mosquitto - ${CLUSTER_NAME}" + target: + creationPolicy: Owner + deletionPolicy: Retain + name: *name + template: + type: Opaque + data: + passwd: | + {{ .X_EMQX_MQTT_BRIDGE_USERNAME }}:{{ .X_EMQX_MQTT_BRIDGE_PASSWORD }} + {{ .X_EMQX_MQTT_HASS_USERNAME }}:{{ .X_EMQX_MQTT_HASS_PASSWORD }} + {{ .X_EMQX_MQTT_Z2M_USERNAME }}:{{ .X_EMQX_MQTT_Z2M_PASSWORD }} + {{ .X_EMQX_MQTT_VALETUDO_USERNAME }}:{{ .X_EMQX_MQTT_VALETUDO_PASSWORD }} + {{ .X_EMQX_MQTT_ESPHOME_CN105_USERNAME }}:{{ .X_EMQX_MQTT_ESPHOME_CN105_PASSWORD }} + acl.conf: | + user {{ .X_EMQX_MQTT_BRIDGE_USERNAME }} + topic readwrite # + + user {{ .X_EMQX_MQTT_HASS_USERNAME }} + topic readwrite homeassistant/# + topic readwrite hass/# + topic readwrite zigbee2mqtt/# + topic readwrite valetudo/# + + user {{ .X_EMQX_MQTT_Z2M_USERNAME }} + topic write homeassistant/# + topic read homeassistant/status + topic readwrite zigbee2mqtt/# + + user {{ .X_EMQX_MQTT_VALETUDO_USERNAME }} + topic write homeassistant/# + topic read homeassistant/status + topic readwrite valetudo/# + + user {{ .X_EMQX_MQTT_ESPHOME_CN105_USERNAME }} + topic write homeassistant/# + topic read homeassistant/status + topic readwrite esphome/# + topic readwrite cn105/+/# + bridge-1.conf: | + connection mosquitto-0-bridge + address mosquitto-0.mosquitto-bridge.mosquitto.svc.cluster.local:2883 + topic # both 0 + local_username {{ .X_EMQX_MQTT_BRIDGE_USERNAME }} + local_password {{ .X_EMQX_MQTT_BRIDGE_PASSWORD }} + local_clientid mosquitto-0-bridge + remote_username {{ .X_EMQX_MQTT_BRIDGE_USERNAME }} + remote_password {{ .X_EMQX_MQTT_BRIDGE_PASSWORD }} + remote_clientid mosquitto-1-bridge + try_private true + bridge_protocol_version mqttv50 diff --git a/kube/deploy/core/db/mosquitto/app/hr.yaml b/kube/deploy/core/db/mosquitto/app/hr.yaml new file mode 100644 index 00000000..9f3b1da7 --- /dev/null +++ b/kube/deploy/core/db/mosquitto/app/hr.yaml @@ -0,0 +1,168 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/app-template-4.1.2/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: &app mosquitto + namespace: *app +spec: + interval: 5m + chart: + spec: + chart: app-template + version: 4.1.2 + sourceRef: + name: bjw-s + kind: HelmRepository + namespace: flux-system + values: + controllers: + app: + type: statefulset + replicas: 2 + containers: + app: + image: &img + repository: public.ecr.aws/docker/library/eclipse-mosquitto + tag: 2.0.22@sha256:d219d3a72847f3aed6a1d66975972d3b17f86e39e8f6f6b86b4088b879c1a2d6 + command: ["sh", "-c"] + args: + - exec mosquitto -c "/config/$(hostname).conf" + env: &env + TZ: "${CONFIG_TZ}" + securityContext: &sc + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + cpu: "1" + memory: 256Mi + probes: + liveness: + enabled: true + custom: true + spec: + exec: + command: + - /bin/sh + - -c + - >- + /usr/bin/mosquitto_pub -h localhost -p 1883 -t mosquitto/healthcheck/$(hostname)/liveness -m "ok $(date)" -r -q 0 + readiness: + enabled: true + service: + app: + controller: app + type: LoadBalancer + annotations: + coredns.io/hostname: "${APP_DNS_MOSQUITTO:=mosquitto}" + lbipam.cilium.io/ips: "${APP_IP_MOSQUITTO:=127.0.0.1}" + ports: + # mqtt: + # port: 1883 + # protocol: TCP + # appProtocol: mqtt + tls: + port: 8883 + protocol: TCP + appProtocol: secure-mqtt + bridge: + primary: false + controller: app + ports: + http: + port: 2883 + protocol: TCP + appProtocol: mqtt + persistence: + config: + type: configMap + identifier: config + globalMounts: + - path: /config + secrets: + type: secret + name: mosquitto-secrets + defaultMode: 0400 + globalMounts: + - path: /secrets + tls: + type: secret + name: short-domain-tls + defaultMode: 0400 + globalMounts: + - path: /tls + configMaps: + config: + data: + common.conf: | + log_type error + log_type warning + + allow_anonymous false + per_listener_settings false + password_file /secrets/passwd + acl_file /secrets/acl.conf + + listener 1883 + protocol mqtt + use_username_as_clientid true + + listener 8883 + protocol mqtt + use_username_as_clientid true + certfile /tls/tls.crt + keyfile /tls/tls.key + + listener 2883 + protocol mqtt + use_username_as_clientid false + mosquitto-0.conf: | + include_dir /config/common + mosquitto-1.conf: | + include_dir /config/common + include_dir /secrets/bridge + defaultPodOptions: + automountServiceAccountToken: false + enableServiceLinks: false + hostAliases: + - ip: "${APP_IP_AUTHENTIK:=127.0.0.1}" + hostnames: ["${APP_DNS_AUTHENTIK:=authentik}"] + dnsConfig: + options: + - name: ndots + value: "1" + hostUsers: false + securityContext: + runAsNonRoot: true + runAsUser: &uid 1000 + runAsGroup: *uid + fsGroup: *uid + fsGroupChangePolicy: Always + seccompProfile: { type: "RuntimeDefault" } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: "{{ .Release.Name }}" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "fuckoff.home.arpa/{{ .Release.Name }}" + operator: DoesNotExist + networkpolicies: + same-ns: + controller: app + policyTypes: [Ingress, Egress] + rules: + ingress: [from: [{podSelector: {}}]] + egress: [to: [{podSelector: {}}]] diff --git a/kube/deploy/core/db/emqx/cluster/netpol.yaml b/kube/deploy/core/db/mosquitto/app/netpol.yaml similarity index 65% rename from kube/deploy/core/db/emqx/cluster/netpol.yaml rename to kube/deploy/core/db/mosquitto/app/netpol.yaml index 7fedc13e..ae539542 100644 --- a/kube/deploy/core/db/emqx/cluster/netpol.yaml +++ b/kube/deploy/core/db/mosquitto/app/netpol.yaml @@ -3,59 +3,51 @@ apiVersion: cilium.io/v2 kind: CiliumClusterwideNetworkPolicy metadata: - name: "apps-to-emqx" + name: "apps-to-mosquitto" spec: endpointSelector: matchLabels: - db.home.arpa/emqx: "emqx" + db.home.arpa/mqtt: "allow" egress: - - toEndpoints: &emqx + - toEndpoints: &mosquitto - matchLabels: - io.kubernetes.pod.namespace: emqx - apps.emqx.io/instance: emqx + io.kubernetes.pod.namespace: mosquitto + app.kubernetes.io/name: mosquitto toPorts: - ports: - port: "8883" protocol: TCP - - toEndpoints: *emqx + - toEndpoints: *mosquitto icmps: [{}] --- # yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json apiVersion: cilium.io/v2 kind: CiliumNetworkPolicy metadata: - name: &app emqx + name: &app mosquitto namespace: *app spec: endpointSelector: &cluster matchLabels: - apps.emqx.io/instance: emqx + app.kubernetes.io/name: mosquitto ingress: - # EMQX operator (readiness gate) - - fromEndpoints: - - matchLabels: - app.kubernetes.io/name: emqx-operator - # same EMQX cluster + # same mosquitto cluster - fromEndpoints: [*cluster] # labelled pods - fromEndpoints: &labelled - matchExpressions: - key: io.kubernetes.pod.namespace operator: Exists - - key: db.home.arpa/emqx + - key: db.home.arpa/mqtt operator: In - values: [emqx] + values: [allow] toPorts: &ports - ports: - port: "8883" protocol: TCP - fromEndpoints: *labelled icmps: [{}] - # operator - - fromEndpoints: - - matchLabels: - io.kubernetes.pod.namespace: emqx - app.kubernetes.io/name: emqx-operator + # Valetudo - fromCIDRSet: - cidr: "${IP_ROBOROCK}/32" toPorts: *ports diff --git a/kube/deploy/core/db/mosquitto/ks.yaml b/kube/deploy/core/db/mosquitto/ks.yaml new file mode 100644 index 00000000..24feeb0f --- /dev/null +++ b/kube/deploy/core/db/mosquitto/ks.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: mosquitto-app + namespace: flux-system + labels: &l + app.kubernetes.io/name: "mosquitto" +spec: + targetNamespace: "mosquitto" + commonMetadata: + labels: *l + path: ./kube/deploy/core/db/mosquitto/app + components: + - ../../../../core/flux-system/alerts/template/ + dependsOn: + - name: crds + namespace: flux-system diff --git a/kube/deploy/core/db/emqx/kustomization.yaml b/kube/deploy/core/db/mosquitto/kustomization.yaml similarity index 100% rename from kube/deploy/core/db/emqx/kustomization.yaml rename to kube/deploy/core/db/mosquitto/kustomization.yaml diff --git a/kube/deploy/core/db/emqx/ns.yaml b/kube/deploy/core/db/mosquitto/ns.yaml similarity index 65% rename from kube/deploy/core/db/emqx/ns.yaml rename to kube/deploy/core/db/mosquitto/ns.yaml index 6a7d3726..2f29f917 100644 --- a/kube/deploy/core/db/emqx/ns.yaml +++ b/kube/deploy/core/db/mosquitto/ns.yaml @@ -2,9 +2,9 @@ apiVersion: v1 kind: Namespace metadata: - name: emqx + name: mosquitto labels: kustomize.toolkit.fluxcd.io/prune: disabled - pod-security.kubernetes.io/enforce: &ps baseline # operator securityContext is meh + pod-security.kubernetes.io/enforce: &ps restricted pod-security.kubernetes.io/audit: *ps pod-security.kubernetes.io/warn: *ps