feat: make kube-apiserver endpoint configurable

Add helm values to configure kube-apiserver endpoint when using type: DaemonSet

Signed-off-by: Sander ter Schure <sander.terschure@gmail.com>

Makefile

@@ -122,11 +122,11 @@ docs:
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-edge.yml
 	helm template -n kube-system talos-cloud-controller-manager \
 		--set-string image.tag=$(TAG) \
-		--set useDaemonSet=true \
+		--set daemonSet.enabled=true \
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset.yml
 	helm template -n kube-system talos-cloud-controller-manager \
 		-f charts/talos-cloud-controller-manager/values.edge.yaml \
-		--set useDaemonSet=true \
+		--set daemonSet.enabled=true \
 		charts/talos-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset-edge.yml
 	helm-docs charts/talos-cloud-controller-manager
 

charts/talos-cloud-controller-manager/README.md

@@ -82,6 +82,9 @@ helm upgrade -i --namespace=kube-system -f talos-ccm.yaml \
 | Key | Type | Default | Description |
 |-----|------|---------|-------------|
 | affinity | object | `{}` | Affinity for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity |
+| daemonSet | object | `{"enabled":false,"k8s":{"serviceHost":"","servicePort":6443}}` | Deploy CCM  in Daemonset mode. CCM will use hostNetwork and connect to the Kubernetes API server on the current node by default. Optionally you can specify the Kubernetes API server host and port. You can run it without CNI plugin. |
+| daemonSet.k8s.serviceHost | string | `""` | Kubernetes API server host. Default is the current node IP. |
+| daemonSet.k8s.servicePort | int | `6443` | Kubernetes API server port. Default is 6443. |
 | enabledControllers | list | `["cloud-node","node-csr-approval"]` | List of controllers should be enabled. Use '*' to enable all controllers. Support only `cloud-node, cloud-node-lifecycle, node-csr-approval, node-ipam-controller` controllers. |
 | extraArgs | list | `[]` | Any extra arguments for talos-cloud-controller-manager |
 | fullnameOverride | string | `""` | String to fully override deployment name. |
@@ -108,4 +111,3 @@ helm upgrade -i --namespace=kube-system -f talos-ccm.yaml \
 | tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ |
 | transformations | list | `[]` | List of node transformations. Available matchExpressions key values: https://github.com/siderolabs/talos/blob/main/pkg/machinery/resources/runtime/platform_metadata.go#L28 |
 | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update stategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment |
-| useDaemonSet | bool | `false` | Deploy CCM  in Daemonset mode. CCM will use hostNetwork and current node to access kubernetes/talos API You can run it without CNI plugin. |

charts/talos-cloud-controller-manager/templates/deployment.yaml

@@ -1,5 +1,5 @@
 apiVersion: apps/v1
-{{- if .Values.useDaemonSet }}
+{{- if .Values.daemonSet.enabled }}
 kind: DaemonSet
 {{- else }}
 kind: Deployment
@@ -10,7 +10,7 @@ metadata:
     {{- include "talos-cloud-controller-manager.labels" . | nindent 4 }}
   namespace: {{ .Release.Namespace }}
 spec:
-  {{- if not .Values.useDaemonSet }}
+  {{- if not .Values.daemonSet.enabled }}
   replicas: {{ .Values.replicaCount }}
   strategy:
     type: {{ .Values.updateStrategy.type }}
@@ -37,7 +37,7 @@ spec:
       serviceAccountName: {{ include "talos-cloud-controller-manager.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
-      {{- if .Values.useDaemonSet }}
+      {{- if .Values.daemonSet.enabled }}
       dnsPolicy: ClusterFirstWithHostNet
       hostNetwork: true
       {{- end }}
@@ -61,18 +61,22 @@ spec:
           {{- with .Values.extraArgs }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
-          {{- if .Values.useDaemonSet }}
+          {{- if .Values.daemonSet.enabled }}
           env:
             - name: TALOS_ENDPOINTS
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
             - name: KUBERNETES_SERVICE_HOST
+              {{- if .Values.daemonSet.k8s.serviceHost }}
+              value: {{ .Values.daemonSet.k8s.serviceHost }}
+              {{- else }}
               valueFrom:
                 fieldRef:
                   fieldPath: status.podIP
+              {{- end }}
             - name: KUBERNETES_SERVICE_PORT
-              value: "6443"
+              value: {{ quote .Values.daemonSet.k8s.servicePort }}
           {{- end }}
           ports:
             - name: metrics
@@ -107,7 +111,7 @@ spec:
       {{- with .Values.tolerations }}
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- if .Values.useDaemonSet }}
+      {{- if .Values.daemonSet.enabled }}
         - effect: NoSchedule
           key: node.kubernetes.io/not-ready
           operator: Exists

charts/talos-cloud-controller-manager/values-example.yaml

@@ -1,4 +1,3 @@
-
 # Use latest Talos image
 image:
   pullPolicy: Always
@@ -53,7 +52,8 @@ transformations:
       node-role.kubernetes.io/db: ""
 
 # Deploy the Talos Cloud Controller Manager as a DaemonSet
-useDaemonSet: true
+daemonSet:
+  enabled: true
 
 # Tolerate all taints
 tolerations:

charts/talos-cloud-controller-manager/values.yaml

@@ -125,9 +125,16 @@ resources:
     memory: 64Mi
 
 # -- Deploy CCM  in Daemonset mode.
-# CCM will use hostNetwork and current node to access kubernetes/talos API
+# CCM will use hostNetwork and connect to the Kubernetes API server on the current node by default.
+# Optionally you can specify the Kubernetes API server host and port.
 # You can run it without CNI plugin.
-useDaemonSet: false
+daemonSet:
+  enabled: false
+  k8s:
+    # -- Kubernetes API server host. Default is the current node IP.
+    serviceHost: ""
+    # -- Kubernetes API server port. Default is 6443.
+    servicePort: 6443
 
 # -- Deployment update stategy type.
 # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment