feat: arm support

2025-10-30 17:58:32 +00:00 · 2023-05-05 14:16:23 +03:00
parent 3fb5c0f78e
commit 93d921c1c1
10 changed files with 383 additions and 238 deletions
--- a/_deployments/Makefile
+++ b/_deployments/Makefile
@@ -12,6 +12,9 @@ create-deployments: ## create templates
 	helm template --namespace=kube-system  --version=1.12.7 -f vars/cilium.yaml cilium \
 		cilium/cilium > vars/cilium-result.yaml

+	# helm template --namespace=kube-system -f vars/talos-cloud-controller-manager.yaml talos-cloud-controller-manager \
+	# 	~/work/sergelogvinov/talos-cloud-controller-manager/charts/talos-cloud-controller-manager > vars/talos-cloud-controller-manager-result.yaml
+
 	helm template --namespace=kube-system -f vars/metrics-server.yaml metrics-server \
 		metrics-server/metrics-server > vars/metrics-server-result.yaml

--- a/_deployments/vars/talos-cloud-controller-manager-result.yaml
+++ b/_deployments/vars/talos-cloud-controller-manager-result.yaml
@@ -0,0 +1,273 @@
+---
+# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: talos.dev/v1alpha1
+kind: ServiceAccount
+metadata:
+  name: talos-cloud-controller-manager-talos-secrets
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  roles:
+    - os:reader
+---
+# Source: talos-cloud-controller-manager/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+data:
+  ccm-config.yaml: |
+    global:
+      approveNodeCSR: true
+---
+# Source: talos-cloud-controller-manager/templates/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  verbs:
+  - approve
+---
+# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:talos-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:talos-cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: talos-cloud-controller-manager
+  namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:talos-cloud-controller-manager:extension-apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: talos-cloud-controller-manager
+    namespace: kube-system
+---
+# Source: talos-cloud-controller-manager/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  clusterIP: None
+  type: ClusterIP
+  ports:
+    - name: https
+      port: 50258
+      targetPort: 50258
+      protocol: TCP
+  selector:
+        app.kubernetes.io/name: talos-cloud-controller-manager
+        app.kubernetes.io/instance: talos-cloud-controller-manager
+---
+# Source: talos-cloud-controller-manager/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: talos-cloud-controller-manager
+  labels:
+    helm.sh/chart: talos-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: talos-cloud-controller-manager
+    app.kubernetes.io/instance: talos-cloud-controller-manager
+    app.kubernetes.io/version: "1.3.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: talos-cloud-controller-manager
+      app.kubernetes.io/instance: talos-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: talos-cloud-controller-manager
+        app.kubernetes.io/instance: talos-cloud-controller-manager
+    spec:
+      serviceAccountName: talos-cloud-controller-manager
+      securityContext:
+        fsGroup: 10258
+        fsGroupChangePolicy: OnRootMismatch
+        runAsGroup: 10258
+        runAsNonRoot: true
+        runAsUser: 10258
+      containers:
+        - name: talos-cloud-controller-manager
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          image: "ghcr.io/siderolabs/talos-cloud-controller-manager:edge"
+          imagePullPolicy: Always
+          command: ["/talos-cloud-controller-manager"]
+          args:
+            - --v=4
+            - --cloud-provider=talos
+            - --cloud-config=/etc/talos/ccm-config.yaml
+            - --controllers=cloud-node
+            - --leader-elect-resource-name=cloud-controller-manager-talos
+            - --use-service-account-credentials
+            - --secure-port=50258
+          ports:
+            - containerPort: 50258
+              name: https
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: https
+              scheme: HTTPS
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 10m
+              memory: 64Mi
+          volumeMounts:
+            - name: cloud-config
+              mountPath: /etc/talos
+              readOnly: true
+            - name: talos-secrets
+              mountPath: /var/run/secrets/talos.dev
+              readOnly: true
+      nodeSelector:
+        node-role.kubernetes.io/control-plane: ""
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+        - effect: NoSchedule
+          key: node.cloudprovider.kubernetes.io/uninitialized
+          operator: Exists
+      volumes:
+        - name: cloud-config
+          configMap:
+            name: talos-cloud-controller-manager
+            defaultMode: 416 # 0640
+        - name: talos-secrets
+          secret:
+            secretName: talos-cloud-controller-manager-talos-secrets
+            defaultMode: 416 # 0640
--- a/_deployments/vars/talos-cloud-controller-manager.yaml
+++ b/_deployments/vars/talos-cloud-controller-manager.yaml
@@ -0,0 +1,6 @@
+
+image:
+  pullPolicy: Always
+  tag: edge
+
+logVerbosityLevel: 4
--- a/hetzner/Makefile
+++ b/hetzner/Makefile
@@ -60,3 +60,11 @@ create-secrets:
 	dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret
 	kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret
 	rm -f hcloud-csi-secret.secret
+
+helm-repos: ## add helm repos
+	helm repo add hcloud               https://charts.hetzner.cloud
+	helm repo update
+
+create-deployments:
+	helm template --namespace=kube-system -f deployments/hcloud-cloud-controller-manager.yaml \
+		hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml
--- a/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml
+++ b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml
@@ -0,0 +1,88 @@
+---
+# Source: hcloud-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cloud-controller-manager
+  namespace: kube-system
+---
+# Source: hcloud-cloud-controller-manager/templates/clusterrolebinding.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: cloud-controller-manager
+    namespace: kube-system
+---
+# Source: hcloud-cloud-controller-manager/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hcloud-cloud-controller-manager
+  namespace: kube-system
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: 'hcloud-cloud-controller-manager'
+      app.kubernetes.io/name: 'hcloud-cloud-controller-manager'
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: 'hcloud-cloud-controller-manager'
+        app.kubernetes.io/name: 'hcloud-cloud-controller-manager'
+    spec:
+      serviceAccountName: cloud-controller-manager
+      dnsPolicy: Default
+      tolerations:
+        # Allow HCCM itself to schedule on nodes that have not yet been initialized by HCCM.
+        - key: "node.cloudprovider.kubernetes.io/uninitialized"
+          value: "true"
+          effect: "NoSchedule"
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+
+        # Allow HCCM to schedule on control plane nodes.
+        - key: "node-role.kubernetes.io/master"
+          effect: NoSchedule
+          operator: Exists
+        - key: "node-role.kubernetes.io/control-plane"
+          effect: NoSchedule
+          operator: Exists
+
+        - key: "node.kubernetes.io/not-ready"
+          effect: "NoExecute"
+      containers:
+        - name: hcloud-cloud-controller-manager
+          command:
+            - "/bin/hcloud-cloud-controller-manager"
+            - "--allow-untagged-cloud"
+            - "--cloud-provider=hcloud"
+            - "--leader-elect=false"
+            - "--route-reconciliation-period=30s"
+          env:
+            - name: HCLOUD_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  key: token
+                  name: hcloud
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: HCLOUD_METRICS_ENABLED
+              value: "false"
+          image: hetznercloud/hcloud-cloud-controller-manager:v1.15.0
+          ports:
+          resources:
+            requests:
+              cpu: 100m
+              memory: 50Mi
+      priorityClassName: system-cluster-critical
--- a/hetzner/deployments/hcloud-cloud-controller-manager.yaml
+++ b/hetzner/deployments/hcloud-cloud-controller-manager.yaml
@@ -48,8 +48,7 @@ spec:
        - key: "node-role.kubernetes.io/control-plane"
          effect: NoSchedule
      containers:
-        - image: hetznercloud/hcloud-cloud-controller-manager:v1.13.2
-        # - image: ghcr.io/sergelogvinov/hetzner-cloud-controller-manager:v1.13.2-dev
+        - image: hetznercloud/hcloud-cloud-controller-manager:v1.15.0
          name: hcloud-cloud-controller-manager
          args:
            - --cloud-provider=hcloud
--- a/hetzner/deployments/hcloud-csi.yaml
+++ b/hetzner/deployments/hcloud-csi.yaml
@@ -270,7 +270,7 @@ spec:
            secretKeyRef:
              key: token
              name: hcloud
-        image: hetznercloud/hcloud-csi-driver:2.2.0
+        image: hetznercloud/hcloud-csi-driver:v2.3.2
        imagePullPolicy: Always
        livenessProbe:
          failureThreshold: 5
@@ -341,7 +341,7 @@ spec:
          value: 0.0.0.0:9189
        - name: ENABLE_METRICS
          value: "true"
-        image: hetznercloud/hcloud-csi-driver:2.2.0
+        image: hetznercloud/hcloud-csi-driver:v2.3.2
        imagePullPolicy: Always
        livenessProbe:
          failureThreshold: 5
--- a/hetzner/deployments/kubelet-serving-cert-approver.yaml
+++ b/hetzner/deployments/kubelet-serving-cert-approver.yaml
@@ -1,231 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
---
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: certificates:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - certificates.k8s.io
-  resources:
-  - certificatesigningrequests
-  verbs:
-  - get
-  - list
-  - watch
- apiGroups:
-  - certificates.k8s.io
-  resources:
-  - certificatesigningrequests/approval
-  verbs:
-  - update
- apiGroups:
-  - authorization.k8s.io
-  resources:
-  - subjectaccessreviews
-  verbs:
-  - create
- apiGroups:
-  - certificates.k8s.io
-  resourceNames:
-  - kubernetes.io/kubelet-serving
-  resources:
-  - signers
-  verbs:
-  - approve
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: events:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - patch
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: psp:kubelet-serving-cert-approver
-rules:
- apiGroups:
-  - policy
-  resourceNames:
-  - kubelet-serving-cert-approver
-  resources:
-  - podsecuritypolicies
-  verbs:
-  - use
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: events:kubelet-serving-cert-approver
-  namespace: default
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: events:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: psp:kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: psp:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: certificates:kubelet-serving-cert-approver
-subjects:
- kind: ServiceAccount
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
---
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-spec:
-  ports:
-  - name: metrics
-    port: 9090
-    protocol: TCP
-    targetPort: metrics
-  selector:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  labels:
-    app.kubernetes.io/instance: kubelet-serving-cert-approver
-    app.kubernetes.io/name: kubelet-serving-cert-approver
-  name: kubelet-serving-cert-approver
-  namespace: kubelet-serving-cert-approver
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/instance: kubelet-serving-cert-approver
-      app.kubernetes.io/name: kubelet-serving-cert-approver
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/instance: kubelet-serving-cert-approver
-        app.kubernetes.io/name: kubelet-serving-cert-approver
-    spec:
-      nodeSelector:
-        node-role.kubernetes.io/control-plane: ""
-      tolerations:
-      - effect: NoSchedule
-        key: node-role.kubernetes.io/control-plane
-        operator: Exists
-      - effect: NoSchedule
-        key: node.cloudprovider.kubernetes.io/uninitialized
-        operator: Exists
-      containers:
-      - args:
-        - serve
-        env:
-        - name: NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        image: ghcr.io/alex1989hu/kubelet-serving-cert-approver:main
-        imagePullPolicy: IfNotPresent
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: health
-          initialDelaySeconds: 6
-        name: cert-approver
-        ports:
-        - containerPort: 8080
-          name: health
-        - containerPort: 9090
-          name: metrics
-        readinessProbe:
-          httpGet:
-            path: /readyz
-            port: health
-          initialDelaySeconds: 3
-        resources:
-          limits:
-            cpu: 250m
-            memory: 32Mi
-          requests:
-            cpu: 10m
-            memory: 16Mi
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          privileged: false
-          readOnlyRootFilesystem: true
-          runAsNonRoot: true
-      priorityClassName: system-cluster-critical
-      securityContext:
-        fsGroup: 65534
-        runAsGroup: 65534
-        runAsUser: 65534
-      serviceAccountName: kubelet-serving-cert-approver
--- a/hetzner/images/variables.pkr.hcl
+++ b/hetzner/images/variables.pkr.hcl
@@ -17,7 +17,7 @@ variable "hcloud_type" {

 variable "talos_version" {
  type    = string
-  default = "v1.4.0-beta.1"
+  default = "v1.4.1"
 }

 locals {
--- a/hetzner/templates/controlplane.yaml.tpl
+++ b/hetzner/templates/controlplane.yaml.tpl
@@ -114,10 +114,9 @@ cluster:
  externalCloudProvider:
    enabled: true
    manifests:
-      - https://raw.githubusercontent.com/siderolabs/talos-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml
      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml
      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-csi.yaml
-      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/kubelet-serving-cert-approver.yaml
      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml
      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml
      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml