feat(tailscale): add subnet routers

2026-03-21 13:39:44 +00:00 · 2023-12-17 02:14:08 +08:00
parent dcdbe79f9c
commit 784d45362e
5 changed files with 175 additions and 2 deletions
--- a/kube/clusters/biohazard/config/vars.sops.env
+++ b/kube/clusters/biohazard/config/vars.sops.env
@@ -16,6 +16,7 @@ USERS_3_NAME=ENC[AES256_GCM,data:BxSWnRnQwXfHqg==,iv:JmzuZmZZnuQnhI9SYt1TBmBLojm
 ASN_CLUSTER=ENC[AES256_GCM,data:745TiWY=,iv:u/ipc6hOnnF3jy9qwo0ol3fenAt2KocFdlJ5scYpHr8=,tag:W1hzY16Y6CF0I33060ajxg==,type:str]
 ASN_ROUTER=ENC[AES256_GCM,data:uaoV8Ng=,iv:65f9Mym3J9OkYZA0m+y/Ffgr3aMU3cF9LmUg2qXExmU=,tag:kELo3H6hWh86D7PN4Y7ytQ==,type:str]
 ASN_EC2_INGRESS=ENC[AES256_GCM,data:YhsD6Bk=,iv:lgQlCo5CIaZS4klgTJppwNnbXkPhq3S2W+G85C3CRfQ=,tag:Y4xQ1hXOGN7D3HCQ9e65PA==,type:str]
+ASN_CLUSTER_NODES=ENC[AES256_GCM,data:dvp/EhY=,iv:YlQ3Y4Ne6E5RuOMOHITY/Pdpu6L89Kg0rk8XqBjbt1Q=,tag:tFFZ7VAhGc5faWhc2er4yA==,type:str]
 IP_ROUTER_LAN=ENC[AES256_GCM,data:q+9MIIuBLPA=,iv:pzWM3e0qgyRLgYtXv3aoKqX6ZOnpQURGBWaLZZRfQGc=,tag:xEiU2fV3Wt0YHd60hALsUQ==,type:str]
 IP_ROUTER_LAN_CIDR=ENC[AES256_GCM,data:VBNZEYACQMQduOU=,iv:is1RkkLkgUYuNPypTFRm7krP9nb1rkrZ64pkQT+5LEM=,tag:opkUbEo8JR1Gp13pklKz7g==,type:str]
 IP_ROUTER_VLAN_K8S=ENC[AES256_GCM,data:BF7rMLUGyiMb,iv:H+s1v1sl6ZNJEvF1QO5kIYE7jquhLrDXbPnpE2PywUY=,tag:Sux+8RhfEHfZDXT2z4S5Jw==,type:str]
@@ -177,6 +178,9 @@ APP_UID_GROCY=ENC[AES256_GCM,data:5oe0G/I=,iv:PA/TNw3G9HWEiKs6Rea0MDhFSjxrS25IbP
 APP_DNS_NEXTCLOUD=ENC[AES256_GCM,data:DGppBt9YjJGQ,iv:vtA3/8jSQxbDNjHCJG0y5xygnhddg5sEC2IY33GXEkc=,tag:eC+9MVXyGMW+mgtQOAENZQ==,type:str]
 APP_UID_NEXTCLOUD=ENC[AES256_GCM,data:DKU=,iv:oaQPXmjTY33+QViY8RQAsbcMIQNqXnHTseseHZuPhgo=,tag:57SRZJxX84lrbRD0bok24g==,type:str]
 APP_DNS_NFS_WEB=ENC[AES256_GCM,data:JCgLEF4O+IGrKK54Sw2f,iv:1lX21EAq/2U+5SCmWSzPuAckapSiRu3v/V2fUjzN4Rg=,tag:afooloE/hSnPoYakX7I60g==,type:str]
+APP_IP_TAILSCALE=ENC[AES256_GCM,data:Qe3K9HQqMlIQEBc=,iv:TKU933gQCfVxGQrcn3ck8NFpGLtCATxo4HjYeGUWE2Q=,tag:yQlebPVcLEsaV/pi/woqWw==,type:str]
+APP_UID_TAILSCALE=ENC[AES256_GCM,data:HpYp/hQ=,iv:3WmuUMZoq9bGSdEG087iO1WQhZ9GIaMqrQHLGjIebsU=,tag:1h1I+dCNZACauDW7LSB/Jw==,type:str]
+CONFIG_TAILSCALE_NODE_PORT=ENC[AES256_GCM,data:5fOGZnU=,iv:ACISp8g5R65r4wfL9GPCenCqqszwalLiAa99BDVWS7w=,tag:ECJ5gRru2kd8ccGXEbj7yQ==,type:str]
 CONFIG_MINECRAFT_OPS=ENC[AES256_GCM,data:al3glJDrtuqtTM2z4W7n+tPNf6XVfK64Jdb9s5RAE5NUwxyK,iv:kYqlsOabsa2iBZKgqjOpFYJo0DMFuoo3ZWCqb/Xzi5c=,tag:nIqPXvBvxdi8crMj1CYsEw==,type:str]
 CONFIG_MINECRAFT_ICON=ENC[AES256_GCM,data:nNzsyRclLnPZ+8Td/WJg2u8V/QKf/xowrghmTaKRNb9a5BMOxtzmiyAt6Us8OoY=,iv:b7fHZQdOjc4oCCLtLhopNg6G7IS2u9NUdBLCN6CjSKc=,tag:+cPgP1oK/9+EK2tB9Y45zw==,type:str]
 CONFIG_MINECRAFT_NAME=ENC[AES256_GCM,data:1qSqJGmGON9BhJKRJA==,iv:Sdwq0LLLdBQlr3m+0Ey2IE9FcRtVKOtXsswLMMp9A5A=,tag:WpaTzqSO3+N+vnJkGI+pCQ==,type:str]
@@ -202,8 +206,8 @@ SECRET_TAILSCALE_TALOS_AUTHKEY_CHARLOTTE=ENC[AES256_GCM,data:R99pfS9Nw4UD5drLMxC
 SECRET_TAILSCALE_TALOS_AUTHKEY_CHISE=ENC[AES256_GCM,data:io5oMtjzwQk0+ypUhNOTRrZV9sfcUKKrr5UApBrHXbNX1pCP8W2Tcpl2OoXRb1q2rgdZNQL2k+WS,iv:MpOxyFc+PgNBK11vQMbOc0shKX12LVEvFetfDuIxcvg=,tag:OAd0hGkAviTr+vheEe5EBg==,type:str]
 sops_age__list_0__map_enc=-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFSXFvLzFQaFJ0OVJKUFV5\nTWh2OUltUlpJWFlVVytFYU9VajBHSnQ4SGdjCnRVbEVXdDVyUHJrR05Ba0xvUm1l\nTkt2YmNUZy90ZFA2b3QrODFKZ01EVG8KLS0tIEw2dkd1cnFCbnI5eWxKL2o1aDVB\nN0hveXZ2dWdxQ2k2L0pGR0ROMStVTmsK4dV/hNyDjsYnVUiFQ7kqdmcVHfYyVckz\nh/rwLjcZgsup72WDVP3v6Eul8B3LKFrSb8CDFA54tyQmSdFDCQC+Zg==\n-----END AGE ENCRYPTED FILE-----\n
 sops_age__list_0__map_recipient=age1u57l4s400gqstc0p485j4646cemntufr0pcyp32yudklsp90xpmszxvnkj
-sops_lastmodified=2023-12-14T21:53:00Z
-sops_mac=ENC[AES256_GCM,data:lcBe/+mj/NBiSkFBfMIObIsaoVp7J4NO/5EeNtERtTEhKpy5x11o5JVrjgegrPYJadzSsH5Wrx19IYB8ExbOe2sUg2FY1Ikiz76rrUbi/kQ+isKRQvJIGoG9sVNp3kZcs0Gn8GKm7+Ujs5FL2CxoK3Hq6koN1RWq+0Jb/Mwtoyo=,iv:bZw1FCvjsQdmX0ChBWWY4BMAhQvVs2Uly4EHo/F94P0=,tag:U9o/vkEl2UsFYpyjZ/8yGQ==,type:str]
+sops_lastmodified=2023-12-16T18:09:22Z
+sops_mac=ENC[AES256_GCM,data:2f76C0+p+lFuqAIX5iK+xct6bqrji0p5uTS81L9ulkTVHoBtf0abbugB7AKORc+p7bYdGAb7C+Kc4+xoDh5ANWfOL9SnKLpJk9iN5KAW7mptmCrM0Tffpjrp8FD4CYTReS2CE5wchQ46woHq4emuohKmG0A3a4xCPAxQDjfIZ0M=,iv:1np0W3d8X+9i8Rr8fvU+FXIXEPE2inpZ7cQl0PSXpRE=,tag:IbtUA6V9vpqyZfy5gvO1GQ==,type:str]
 sops_pgp__list_0__map_created_at=2023-06-01T18:01:04Z
 sops_pgp__list_0__map_enc=-----BEGIN PGP MESSAGE-----\n\nhF4DAAAAAAAAAAASAQdAbA35718t0WVKrjQFYUPviCb0lVuh8NpfSdJCHjHcWWww\n8ak4q4VL69tZLSjQHx+VsMmKooknxWz6pw0lGxyDYlZMQ81bodInjaZGFZSz8Uuh\n0l4BhDCNDBBALTrnTliz6/DAHvmavI4UxMHost5alFio9JPkTDNmXZyvcy1/R6aw\n/uhQXLUBRvm0TSOhBZb7d0SLkLfe02Um40w1TibpKXsZz1GOMbPRNBMHHra0QIuQ\n=0jA+\n-----END PGP MESSAGE-----\n
 sops_pgp__list_0__map_fp=31E70E5BC80C58AFF5DD649921AC5A1AC6E5B7F2
--- a/kube/deploy/core/_networking/tailscale/ks.yaml
+++ b/kube/deploy/core/_networking/tailscale/ks.yaml
@@ -6,4 +6,13 @@ metadata:
  namespace: flux-system
 spec:
  path: ./kube/deploy/core/_networking/tailscale/app
+  dependsOn: []
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: 1-core-1-networking-tailscale-router
+  namespace: flux-system
+spec:
+  path: ./kube/deploy/core/_networking/tailscale/router
  dependsOn: []
--- a/kube/deploy/core/_networking/tailscale/router/hr.yaml
+++ b/kube/deploy/core/_networking/tailscale/router/hr.yaml
@@ -0,0 +1,129 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: &app tailscale-router
+  namespace: tailscale
+spec:
+  chart:
+    spec:
+      chart: app-template
+      version: "2.4.0"
+      sourceRef:
+        name: bjw-s
+        kind: HelmRepository
+        namespace: flux-system
+  values:
+    controllers:
+      main:
+        type: "daemonset"
+        containers:
+          main:
+            image: &img
+              repository: "ghcr.io/tailscale/tailscale"
+              tag: "v1.56.0@sha256:ed1f9317d0bab2bc17f6eecc29401479b91c938df48c28b1bd3d3014eba9d013"
+            env:
+              TZ: "${CONFIG_TZ}"
+              PORT: &port "${CONFIG_TAILSCALE_NODE_PORT}"
+              # SA_NAME: "tailscale-router"
+              TS_USERSPACE: "true"
+              # TS_HOSTNAME: &nodeEnv
+              #   valueFrom:
+              #     fieldRef:
+              #       fieldPath: "spec.nodeName"
+              # TS_KUBE_SECRET: *nodeEnv
+              TS_HOSTNAME:
+                valueFrom:
+                  fieldRef:
+                    fieldPath: "metadata.name"
+              TS_KUBE_SECRET: ""
+              TS_AUTHKEY: "file:/authkey"
+              TS_ROUTES: "${IP_ROUTER_VLAN_K8S_CIDR},${IP_LB_CIDR},${IP_SVC_CIDR_V4}"
+              TS_EXTRA_ARGS: "--advertise-exit-node=true --advertise-connector=true --advertise-tags=tag:kube"
+              TS_TAILSCALED_EXTRA_ARGS: "--state=mem: --debug=0.0.0.0:8080 --socks5-server=0.0.0.0:1080 --outbound-http-proxy-listen=0.0.0.0:28081"
+            securityContext:
+              readOnlyRootFilesystem: true
+              allowPrivilegeEscalation: false
+              capabilities:
+                drop: ["ALL"]
+            resources:
+              requests:
+                cpu: "10m"
+                memory: "128Mi"
+              limits:
+                memory: "512Mi"
+    # serviceAccount:
+    #   name: *app
+    #   create: true
+    service:
+      main:
+        ports:
+          http:
+            port: 8080
+          socks5:
+            port: 1080
+            protocol: TCP
+          http-proxy:
+            port: 28081
+            protocol: TCP
+      tailscaled:
+        enabled: true
+        primary: false
+        controller: main
+        type: LoadBalancer
+        annotations:
+          "io.cilium/lb-ipam-ips": "${APP_IP_TAILSCALE}"
+        ports:
+          wireguard:
+            enabled: true
+            port: ${CONFIG_TAILSCALE_NODE_PORT}
+            protocol: UDP
+    persistence:
+      config:
+        enabled: true
+        type: secret
+        name: "tailscale-router-secrets"
+        defaultMode: 0400
+        advancedMounts:
+          main:
+            main:
+              - subPath: "authkey"
+                path: "/authkey"
+                readOnly: true
+      # tmp:
+      #   enabled: true
+      #   type: emptyDir
+      #   medium: Memory
+      #   globalMounts:
+      #     - subPath: "tmp"
+      #       path: "/tmp"
+      #       readOnly: false
+      #     - subPath: "cache"
+      #       path: ".cache"
+      #       readOnly: false
+    defaultPodOptions:
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        runAsNonRoot: false
+        runAsUser: &uid ${APP_UID_TAILSCALE}
+        runAsGroup: *uid
+        fsGroup: *uid
+        fsGroupChangePolicy: Always
+        seccompProfile: {type: "RuntimeDefault"}
+      topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: "kubernetes.io/hostname"
+          whenUnsatisfiable: "DoNotSchedule"
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/name: *app
+    serviceMonitor:
+      main:
+        enabled: true
+        endpoints:
+          - port: http
+            scheme: http
+            path: /debug/metrics
+            interval: 1m
+            scrapeTimeout: 30s
--- a/kube/deploy/core/_networking/tailscale/router/netpol.yaml
+++ b/kube/deploy/core/_networking/tailscale/router/netpol.yaml
@@ -0,0 +1,22 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/datreeio/CRDs-catalog/main/cilium.io/ciliumnetworkpolicy_v2.json
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: &name tailscale-router
+  namespace: &app "tailscale"
+spec:
+  endpointSelector:
+    matchLabels:
+      app.kubernetes.io/name: *name
+  ingress:
+    # same namespace
+    - fromEntities:
+        - world
+        - host
+        - remote-node
+  egress:
+    - toEntities:
+        - host
+        - remote-node
+        - cluster
--- a/kube/deploy/core/_networking/tailscale/router/secrets.yaml
+++ b/kube/deploy/core/_networking/tailscale/router/secrets.yaml
@@ -0,0 +1,9 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: tailscale-router-secrets
+  namespace: tailscale
+type: Opaque
+stringData:
+  authkey: "${SECRET_TAILSCALE_OAUTH_CLIENT_SECRET}?preauthorized=true"