From 7dfc6611a3b7663806e0fedc874e6ca17d88ad56 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sun, 23 Oct 2022 17:28:28 +0300 Subject: [PATCH] node autoscaller --- exoscale/README.md | 0 .../exoscale-cloud-controller-manager.yaml | 52 ++++-- .../exoscale-cluster-autoscaler.yaml | 164 ++++++++++++++++++ exoscale/deployments/local-path-storage.yaml | 3 + exoscale/deployments/test-as.yaml | 40 +++++ exoscale/instances-web.tf | 2 +- exoscale/instances-werker.tf | 2 +- exoscale/templates/controlplane.yaml.tpl | 3 +- 8 files changed, 253 insertions(+), 13 deletions(-) create mode 100644 exoscale/README.md create mode 100644 exoscale/deployments/exoscale-cluster-autoscaler.yaml create mode 100644 exoscale/deployments/test-as.yaml diff --git a/exoscale/README.md b/exoscale/README.md new file mode 100644 index 0000000..e69de29 diff --git a/exoscale/deployments/exoscale-cloud-controller-manager.yaml b/exoscale/deployments/exoscale-cloud-controller-manager.yaml index c0ab538..366f442 100644 --- a/exoscale/deployments/exoscale-cloud-controller-manager.yaml +++ b/exoscale/deployments/exoscale-cloud-controller-manager.yaml @@ -82,24 +82,55 @@ rules: - list - watch - update +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - kubernetes.io/kubelet-serving + verbs: + - approve +- apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: system:cloud-controller-manager + name: system:exoscale-cloud-controller-manager roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:cloud-controller-manager subjects: - kind: ServiceAccount - name: cloud-controller-manager + name: exoscale-cloud-controller-manager namespace: kube-system --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: system:cloud-controller-manager + name: system:exoscale-cloud-controller-manager namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io @@ -107,7 +138,7 @@ roleRef: name: extension-apiserver-authentication-reader subjects: - kind: ServiceAccount - name: cloud-controller-manager + name: exoscale-cloud-controller-manager namespace: kube-system --- apiVersion: apps/v1 @@ -126,9 +157,7 @@ spec: labels: app: exoscale-cloud-controller-manager spec: - dnsPolicy: Default - hostNetwork: true - serviceAccountName: cloud-controller-manager + serviceAccountName: exoscale-cloud-controller-manager nodeSelector: node-role.kubernetes.io/control-plane: "" tolerations: @@ -146,6 +175,12 @@ spec: - --leader-elect=true - --allow-untagged-cloud - --controllers=cloud-node,cloud-node-lifecycle + # env: + # - name: EXOSCALE_SKS_AGENT_RUNNERS + # value: node-csr-validation + envFrom: + - secretRef: + name: exoscale-secret resources: limits: cpu: 500m @@ -153,6 +188,3 @@ spec: requests: cpu: 100m memory: 64Mi - envFrom: - - secretRef: - name: exoscale-secret diff --git a/exoscale/deployments/exoscale-cluster-autoscaler.yaml b/exoscale/deployments/exoscale-cluster-autoscaler.yaml new file mode 100644 index 0000000..7c876ba --- /dev/null +++ b/exoscale/deployments/exoscale-cluster-autoscaler.yaml @@ -0,0 +1,164 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["events", "endpoints"] + verbs: ["create", "patch"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/status"] + verbs: ["update"] + - apiGroups: [""] + resources: ["endpoints"] + resourceNames: ["cluster-autoscaler"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["watch", "list", "get", "update"] + - apiGroups: [""] + resources: + - "namespaces" + - "pods" + - "services" + - "replicationcontrollers" + - "persistentvolumeclaims" + - "persistentvolumes" + verbs: ["watch", "list", "get"] + - apiGroups: ["extensions"] + resources: ["replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["policy"] + resources: ["poddisruptionbudgets"] + verbs: ["watch", "list"] + - apiGroups: ["apps"] + resources: ["statefulsets", "replicasets", "daemonsets"] + verbs: ["watch", "list", "get"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses", "csinodes", "csistoragecapacities", "csidrivers"] + verbs: ["get", "list", "watch"] + - apiGroups: ["batch"] + resources: ["jobs", "cronjobs"] + verbs: ["watch", "list", "get"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resourceNames: ["cluster-autoscaler"] + resources: ["leases"] + verbs: ["get", "update"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["create","list","watch"] + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: + - "cluster-autoscaler-status" + - "cluster-autoscaler-priority-expander" + verbs: ["delete", "get", "update", "watch"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: cluster-autoscaler + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cluster-autoscaler + namespace: kube-system + labels: + k8s-addon: cluster-autoscaler.addons.k8s.io + k8s-app: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cluster-autoscaler +subjects: + - kind: ServiceAccount + name: cluster-autoscaler + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: cluster-autoscaler + name: cluster-autoscaler + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: cluster-autoscaler + template: + metadata: + labels: + app: cluster-autoscaler + spec: + nodeSelector: + node-role.kubernetes.io/master: "" + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cluster-autoscaler + containers: + - image: k8s.gcr.io/autoscaling/cluster-autoscaler:v1.25.0 + imagePullPolicy: IfNotPresent + name: cluster-autoscaler + command: + - ./cluster-autoscaler + - --v=3 + - --logtostderr=true + - --cloud-provider=exoscale + envFrom: + - secretRef: + name: exoscale-secret + resources: + limits: + cpu: 100m + memory: 300Mi + requests: + cpu: 100m + memory: 300Mi diff --git a/exoscale/deployments/local-path-storage.yaml b/exoscale/deployments/local-path-storage.yaml index 3012084..17badc8 100644 --- a/exoscale/deployments/local-path-storage.yaml +++ b/exoscale/deployments/local-path-storage.yaml @@ -83,6 +83,9 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + securityContext: + seccompProfile: + type: RuntimeDefault volumes: - name: config-volume configMap: diff --git a/exoscale/deployments/test-as.yaml b/exoscale/deployments/test-as.yaml new file mode 100644 index 0000000..4206d78 --- /dev/null +++ b/exoscale/deployments/test-as.yaml @@ -0,0 +1,40 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: overprovisioning +value: -1 +globalDefault: false +description: "Priority class used by overprovisioning." +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: overprovisioning + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + run: overprovisioning + template: + metadata: + labels: + run: overprovisioning + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchExpressions: + - key: run + operator: In + values: + - overprovisioning + priorityClassName: overprovisioning + containers: + - name: reserve-resources + image: k8s.gcr.io/pause + resources: + requests: + cpu: "700m" diff --git a/exoscale/instances-web.tf b/exoscale/instances-web.tf index d96aaff..ef24151 100644 --- a/exoscale/instances-web.tf +++ b/exoscale/instances-web.tf @@ -25,6 +25,6 @@ resource "exoscale_instance_pool" "web" { labels = merge(var.tags, { type = "web" }) lifecycle { - ignore_changes = [user_data, labels] + ignore_changes = [size, user_data, labels] } } diff --git a/exoscale/instances-werker.tf b/exoscale/instances-werker.tf index 6b42940..5d426a9 100644 --- a/exoscale/instances-werker.tf +++ b/exoscale/instances-werker.tf @@ -19,7 +19,7 @@ resource "exoscale_instance_pool" "worker" { labels = merge(var.tags, { type = "worker" }) lifecycle { - ignore_changes = [user_data, labels] + ignore_changes = [size, user_data, labels] } } diff --git a/exoscale/templates/controlplane.yaml.tpl b/exoscale/templates/controlplane.yaml.tpl index a608622..f132e6a 100644 --- a/exoscale/templates/controlplane.yaml.tpl +++ b/exoscale/templates/controlplane.yaml.tpl @@ -68,7 +68,6 @@ cluster: warn-version: latest exemptions: namespaces: - - kube-system - ingress-nginx - local-path-provisioner runtimeClasses: [] @@ -99,6 +98,8 @@ cluster: externalCloudProvider: enabled: true manifests: + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/exoscale-cloud-controller-manager.yaml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/exoscale-cluster-autoscaler.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/metrics-server.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/local-path-storage.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/exoscale/deployments/coredns-local.yaml