From 62ff0c0b39943b7fa82588ae2c37e72c4e55bb18 Mon Sep 17 00:00:00 2001 From: Timofei Larkin Date: Tue, 28 Oct 2025 18:28:03 +0300 Subject: [PATCH] [api,lineage] Ensure node-local traffic Since 0.37, many requests to the k8s API now go through a mutating webhook (lineage-controller-webhook). Since the lineage webhook makes multiple requests to the k8s API and, indirectly, to the Cozystack API server, each request for, e.g., creating a secret now causes a lot of chatter between the webhook, the k8s API, and the Cozystack API. When this happens cross-node or, worse yet, cross-zone, this can blow up the latency for simple requests. This patch changes the Cozystack API to a DaemonSet targetting controlplane nodes, configures its service for an `Local` internal traffic policy and adds environment variables indicating that the k8s API server is to be found at :6443, **not only for the Cozystack API, but also for the lineage-controller-webhook.** This is a valid configuration in most scenarios, including the default installation method on top of Talos Linux in Cozystack, however, if this is not valid in your environment, you must now set the values `.lineageControllerWebhook.localK8sAPIEndpoint.enabled` and `.cozystackAPI.localK8sAPIEndpoint.enabled` to `false` in the respective system Helm releases. ```release-note [api,lineage] Configure all chatter between the Lineage webhook, the Cozystack API server and the Kubernetes API server to be confined to a single controlplane node, improving k8s API latency. ``` Signed-off-by: Timofei Larkin --- .../cozystack-api/templates/apiservice.yaml | 6 ++- .../cozystack-api/templates/certmanager.yaml | 45 +++++++++++++++++++ .../cozystack-api/templates/deployment.yaml | 37 ++++++++++++++- .../cozystack-api/templates/service.yaml | 5 ++- packages/system/cozystack-api/values.yaml | 3 ++ .../templates/daemonset.yaml | 10 +++++ .../lineage-controller-webhook/values.yaml | 2 + 7 files changed, 104 insertions(+), 4 deletions(-) create mode 100644 packages/system/cozystack-api/templates/certmanager.yaml diff --git a/packages/system/cozystack-api/templates/apiservice.yaml b/packages/system/cozystack-api/templates/apiservice.yaml index d0ab1185..3cd3665b 100644 --- a/packages/system/cozystack-api/templates/apiservice.yaml +++ b/packages/system/cozystack-api/templates/apiservice.yaml @@ -1,9 +1,10 @@ apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: + annotations: + cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api" name: v1alpha1.apps.cozystack.io spec: - insecureSkipTLSVerify: true group: apps.cozystack.io groupPriorityMinimum: 1000 versionPriority: 15 @@ -15,9 +16,10 @@ spec: apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: + annotations: + cert-manager.io/inject-ca-from: "{{ .Release.Namespace }}/cozystack-api" name: v1alpha1.core.cozystack.io spec: - insecureSkipTLSVerify: true group: core.cozystack.io groupPriorityMinimum: 1000 versionPriority: 15 diff --git a/packages/system/cozystack-api/templates/certmanager.yaml b/packages/system/cozystack-api/templates/certmanager.yaml new file mode 100644 index 00000000..def27bd1 --- /dev/null +++ b/packages/system/cozystack-api/templates/certmanager.yaml @@ -0,0 +1,45 @@ +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: cozystack-api-selfsigned + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cozystack-api-ca + namespace: {{ .Release.Namespace }} +spec: + secretName: cozystack-api-ca + duration: 43800h # 5 years + commonName: cozystack-api-ca + issuerRef: + name: cozystack-api-selfsigned + isCA: true +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: cozystack-api-ca + namespace: {{ .Release.Namespace }} +spec: + ca: + secretName: cozystack-api-ca +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: cozystack-api + namespace: {{ .Release.Namespace }} +spec: + secretName: cozystack-api-cert + duration: 8760h + renewBefore: 720h + issuerRef: + name: cozystack-api-ca + commonName: cozystack-api + dnsNames: + - cozystack-api + - cozystack-api.{{ .Release.Namespace }}.svc diff --git a/packages/system/cozystack-api/templates/deployment.yaml b/packages/system/cozystack-api/templates/deployment.yaml index 46779d2b..1a63a0e0 100644 --- a/packages/system/cozystack-api/templates/deployment.yaml +++ b/packages/system/cozystack-api/templates/deployment.yaml @@ -1,12 +1,18 @@ apiVersion: apps/v1 +{{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} +kind: DaemonSet +{{- else }} kind: Deployment +{{- end }} metadata: name: cozystack-api namespace: cozy-system labels: app: cozystack-api spec: - replicas: 2 + {{- if not .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + replicas: {{ .Values.cozystackAPI.replicas }} + {{- end }} selector: matchLabels: app: cozystack-api @@ -16,6 +22,35 @@ spec: app: cozystack-api spec: serviceAccountName: cozystack-api + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + nodeSelector: + node-role.kubernetes.io/control-plane: "" + {{- end }} containers: - name: cozystack-api + args: + - --tls-cert-file=/tmp/cozystack-api-certs/tls.crt + - --tls-private-key-file=/tmp/cozystack-api-certs/tls.key + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + {{- end }} image: "{{ .Values.cozystackAPI.image }}" + ports: + - containerPort: 443 + name: https + volumeMounts: + - name: cozystack-api-certs + mountPath: /tmp/cozystack-api-certs + readOnly: true + volumes: + - name: cozystack-api-certs + secret: + secretName: cozystack-api-cert + defaultMode: 0400 diff --git a/packages/system/cozystack-api/templates/service.yaml b/packages/system/cozystack-api/templates/service.yaml index 2dcd618b..abe67abc 100644 --- a/packages/system/cozystack-api/templates/service.yaml +++ b/packages/system/cozystack-api/templates/service.yaml @@ -4,9 +4,12 @@ metadata: name: cozystack-api namespace: cozy-system spec: + {{- if .Values.cozystackAPI.localK8sAPIEndpoint.enabled }} + internalTrafficPolicy: Local + {{- end }} ports: - port: 443 protocol: TCP - targetPort: 443 + targetPort: https selector: app: cozystack-api diff --git a/packages/system/cozystack-api/values.yaml b/packages/system/cozystack-api/values.yaml index 1b68eff4..77b4d7a8 100644 --- a/packages/system/cozystack-api/values.yaml +++ b/packages/system/cozystack-api/values.yaml @@ -1,2 +1,5 @@ cozystackAPI: image: ghcr.io/cozystack/cozystack/cozystack-api:v0.37.0@sha256:19d89e8afb90ce38ab7e42ecedfc28402f7c0b56f30957db957c5415132ff6ca + localK8sAPIEndpoint: + enabled: true + replicas: 2 diff --git a/packages/system/lineage-controller-webhook/templates/daemonset.yaml b/packages/system/lineage-controller-webhook/templates/daemonset.yaml index 177bcd8b..22074e1d 100644 --- a/packages/system/lineage-controller-webhook/templates/daemonset.yaml +++ b/packages/system/lineage-controller-webhook/templates/daemonset.yaml @@ -26,6 +26,16 @@ spec: containers: - name: lineage-controller-webhook image: "{{ .Values.lineageControllerWebhook.image }}" + {{- if .Values.lineageControllerWebhook.localK8sAPIEndpoint.enabled }} + env: + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.hostIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + {{- end }} args: {{- if .Values.lineageControllerWebhook.debug }} - --zap-log-level=debug diff --git a/packages/system/lineage-controller-webhook/values.yaml b/packages/system/lineage-controller-webhook/values.yaml index 068de2d6..518ff29c 100644 --- a/packages/system/lineage-controller-webhook/values.yaml +++ b/packages/system/lineage-controller-webhook/values.yaml @@ -1,3 +1,5 @@ lineageControllerWebhook: image: ghcr.io/cozystack/cozystack/lineage-controller-webhook:v0.37.0@sha256:845b8e68cbc277c2303080bcd55597e4334610d396dad258ad56fd906530acc3 debug: false + localK8sAPIEndpoint: + enabled: true