node autoscaller

exoscale/deployments/exoscale-cloud-controller-manager.yaml

@@ -82,24 +82,55 @@ rules:
   - list
   - watch
   - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - certificatesigningrequests/approval
+  verbs:
+  - update
+- apiGroups:
+  - certificates.k8s.io
+  resources:
+  - signers
+  resourceNames:
+  - kubernetes.io/kubelet-serving
+  verbs:
+  - approve
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:cloud-controller-manager
+  name: system:exoscale-cloud-controller-manager
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: system:cloud-controller-manager
 subjects:
 - kind: ServiceAccount
-  name: cloud-controller-manager
+  name: exoscale-cloud-controller-manager
   namespace: kube-system
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:cloud-controller-manager
+  name: system:exoscale-cloud-controller-manager
   namespace: kube-system
 roleRef:
   apiGroup: rbac.authorization.k8s.io
@@ -107,7 +138,7 @@ roleRef:
   name: extension-apiserver-authentication-reader
 subjects:
 - kind: ServiceAccount
-  name: cloud-controller-manager
+  name: exoscale-cloud-controller-manager
   namespace: kube-system
 ---
 apiVersion: apps/v1
@@ -126,9 +157,7 @@ spec:
       labels:
         app: exoscale-cloud-controller-manager
     spec:
-      dnsPolicy: Default
-      hostNetwork: true
-      serviceAccountName: cloud-controller-manager
+      serviceAccountName: exoscale-cloud-controller-manager
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
       tolerations:
@@ -146,6 +175,12 @@ spec:
             - --leader-elect=true
             - --allow-untagged-cloud
             - --controllers=cloud-node,cloud-node-lifecycle
+          # env:
+          #   - name: EXOSCALE_SKS_AGENT_RUNNERS
+          #     value: node-csr-validation
+          envFrom:
+            - secretRef:
+                name: exoscale-secret
           resources:
             limits:
               cpu: 500m
@@ -153,6 +188,3 @@ spec:
             requests:
               cpu: 100m
               memory: 64Mi
-          envFrom:
-            - secretRef:
-                name: exoscale-secret

exoscale/deployments/exoscale-cluster-autoscaler.yaml

@@ -0,0 +1,164 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+  name: cluster-autoscaler
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["events", "endpoints"]
+    verbs: ["create", "patch"]
+  - apiGroups: [""]
+    resources: ["pods/eviction"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["pods/status"]
+    verbs: ["update"]
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    resourceNames: ["cluster-autoscaler"]
+    verbs: ["get", "update"]
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["watch", "list", "get", "update"]
+  - apiGroups: [""]
+    resources:
+      - "namespaces"
+      - "pods"
+      - "services"
+      - "replicationcontrollers"
+      - "persistentvolumeclaims"
+      - "persistentvolumes"
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["extensions"]
+    resources: ["replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["watch", "list"]
+  - apiGroups: ["apps"]
+    resources: ["statefulsets", "replicasets", "daemonsets"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses", "csinodes", "csistoragecapacities", "csidrivers"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["batch"]
+    resources: ["jobs", "cronjobs"]
+    verbs: ["watch", "list", "get"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["create"]
+  - apiGroups: ["coordination.k8s.io"]
+    resourceNames: ["cluster-autoscaler"]
+    resources: ["leases"]
+    verbs: ["get", "update"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create","list","watch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    resourceNames:
+      - "cluster-autoscaler-status"
+      - "cluster-autoscaler-priority-expander"
+    verbs: ["delete", "get", "update", "watch"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cluster-autoscaler
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: cluster-autoscaler
+  namespace: kube-system
+  labels:
+    k8s-addon: cluster-autoscaler.addons.k8s.io
+    k8s-app: cluster-autoscaler
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: cluster-autoscaler
+subjects:
+  - kind: ServiceAccount
+    name: cluster-autoscaler
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: cluster-autoscaler
+  name: cluster-autoscaler
+  namespace: kube-system
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: cluster-autoscaler
+  template:
+    metadata:
+      labels:
+        app: cluster-autoscaler
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+      serviceAccountName: cluster-autoscaler
+      containers:
+        - image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.25.0
+          imagePullPolicy: IfNotPresent
+          name: cluster-autoscaler
+          command:
+            - ./cluster-autoscaler
+            - --v=3
+            - --logtostderr=true
+            - --cloud-provider=exoscale
+          envFrom:
+            - secretRef:
+                name: exoscale-secret
+          resources:
+            limits:
+              cpu: 100m
+              memory: 300Mi
+            requests:
+              cpu: 100m
+              memory: 300Mi

exoscale/deployments/local-path-storage.yaml

@@ -83,6 +83,9 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: metadata.namespace
+          securityContext:
+            seccompProfile:
+              type: RuntimeDefault
       volumes:
         - name: config-volume
           configMap:

exoscale/deployments/test-as.yaml

@@ -0,0 +1,40 @@
+apiVersion: scheduling.k8s.io/v1
+kind: PriorityClass
+metadata:
+  name: overprovisioning
+value: -1
+globalDefault: false
+description: "Priority class used by overprovisioning."
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: overprovisioning
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: overprovisioning
+  template:
+    metadata:
+      labels:
+        run: overprovisioning
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - topologyKey: kubernetes.io/hostname
+              labelSelector:
+                matchExpressions:
+                  - key: run
+                    operator: In
+                    values:
+                      - overprovisioning
+      priorityClassName: overprovisioning
+      containers:
+      - name: reserve-resources
+        image: k8s.gcr.io/pause
+        resources:
+          requests:
+            cpu: "700m"

exoscale/instances-web.tf

@@ -25,6 +25,6 @@ resource "exoscale_instance_pool" "web" {
   labels = merge(var.tags, { type = "web" })
 
   lifecycle {
-    ignore_changes = [user_data, labels]
+    ignore_changes = [size, user_data, labels]
   }
 }

exoscale/instances-werker.tf

@@ -19,7 +19,7 @@ resource "exoscale_instance_pool" "worker" {
   labels = merge(var.tags, { type = "worker" })
 
   lifecycle {
-    ignore_changes = [user_data, labels]
+    ignore_changes = [size, user_data, labels]
   }
 }
 

exoscale/templates/controlplane.yaml.tpl

@@ -68,7 +68,6 @@ cluster:
             warn-version: latest
           exemptions:
             namespaces:
-              - kube-system
               - ingress-nginx
               - local-path-provisioner
             runtimeClasses: []
@@ -99,6 +98,8 @@ cluster:
   externalCloudProvider:
     enabled: true
     manifests:
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/exoscale-cloud-controller-manager.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/exoscale-cluster-autoscaler.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/metrics-server.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/local-path-storage.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/coredns-local.yaml