From ce92b3eef0fd2b4c3a84b096d7897701f94e0d1a Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Sat, 4 May 2024 10:30:14 +0300 Subject: [PATCH] feat(chart): add daemonset mode Deploy CCM in daemonset mode. It's helpful when CNI doesn't have the node.cloudprovider.kubernetes.io/uninitialized taint. Signed-off-by: Serge Logvinov --- Makefile | 4 + .../Chart.yaml | 2 +- .../README.md | 5 +- .../templates/deployment.yaml | 26 ++- .../values.yaml | 5 + .../cloud-controller-manager-daemonset.yml | 202 ++++++++++++++++++ .../deploy/cloud-controller-manager-talos.yml | 8 +- docs/deploy/cloud-controller-manager.yml | 8 +- 8 files changed, 244 insertions(+), 16 deletions(-) create mode 100644 docs/deploy/cloud-controller-manager-daemonset.yml diff --git a/Makefile b/Makefile index 901bc90..227e8ba 100644 --- a/Makefile +++ b/Makefile @@ -112,6 +112,10 @@ docs: -f charts/proxmox-cloud-controller-manager/values.talos.yaml \ --set-string image.tag=$(TAG) \ charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager-talos.yml + helm template -n kube-system proxmox-cloud-controller-manager \ + --set-string image.tag=$(TAG) \ + --set useDaemonSet=true \ + charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager-daemonset.yml helm-docs --sort-values-order=file charts/proxmox-cloud-controller-manager release-update: diff --git a/charts/proxmox-cloud-controller-manager/Chart.yaml b/charts/proxmox-cloud-controller-manager/Chart.yaml index bcddf68..cb35d89 100644 --- a/charts/proxmox-cloud-controller-manager/Chart.yaml +++ b/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -14,7 +14,7 @@ maintainers: # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 +version: 0.2.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. diff --git a/charts/proxmox-cloud-controller-manager/README.md b/charts/proxmox-cloud-controller-manager/README.md index fa14dc6..e22e806 100644 --- a/charts/proxmox-cloud-controller-manager/README.md +++ b/charts/proxmox-cloud-controller-manager/README.md @@ -1,6 +1,6 @@ # proxmox-cloud-controller-manager -![Version: 0.2.0](https://img.shields.io/badge/Version-0.2.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square) +![Version: 0.2.1](https://img.shields.io/badge/Version-0.2.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.4.0](https://img.shields.io/badge/AppVersion-v0.4.0-informational?style=flat-square) A Helm chart for Kubernetes @@ -79,6 +79,7 @@ helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ | podSecurityContext | object | `{"fsGroup":10258,"fsGroupChangePolicy":"OnRootMismatch","runAsGroup":10258,"runAsNonRoot":true,"runAsUser":10258}` | Pods Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | | securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"seccompProfile":{"type":"RuntimeDefault"}}` | Container Security Context. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | | resources | object | `{"requests":{"cpu":"10m","memory":"32Mi"}}` | Resource requests and limits. ref: https://kubernetes.io/docs/user-guide/compute-resources/ | +| useDaemonSet | bool | `false` | Deploy CCM in Daemonset mode. CCM will use hostNetwork. It allows to use CCM without CNI plugins. | | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | Deployment update stategy type. ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment | | nodeSelector | object | `{}` | Node labels for data pods assignment. ref: https://kubernetes.io/docs/user-guide/node-selection/ | | tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","operator":"Exists"}]` | Tolerations for data pods assignment. ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ | @@ -86,5 +87,3 @@ helm upgrade -i --namespace=kube-system -f proxmox-ccm.yaml \ | extraVolumes | list | `[]` | Additional volumes for Pods | | extraVolumeMounts | list | `[]` | Additional volume mounts for Pods | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.12.0](https://github.com/norwoodj/helm-docs/releases/v1.12.0) diff --git a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml index b9ec5c3..2d69040 100644 --- a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml +++ b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -1,14 +1,23 @@ apiVersion: apps/v1 +{{- if .Values.useDaemonSet }} +kind: DaemonSet +{{- else }} kind: Deployment +{{- end }} metadata: name: {{ include "proxmox-cloud-controller-manager.fullname" . }} labels: {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} namespace: {{ .Release.Namespace }} spec: + {{- if not .Values.useDaemonSet }} replicas: {{ .Values.replicaCount }} strategy: type: {{ .Values.updateStrategy.type }} + {{- else }} + updateStrategy: + type: {{ .Values.updateStrategy.type }} + {{- end }} selector: matchLabels: {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} @@ -35,8 +44,14 @@ spec: serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.useDaemonSet }} + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + {{- end }} + {{- with .Values.hostAliases }} hostAliases: - {{- toYaml .Values.hostAliases | nindent 8 }} + {{- toYaml . | nindent 8 }} + {{- end }} initContainers: {{- toYaml .Values.initContainers | nindent 8 }} containers: - name: {{ .Chart.Name }} @@ -89,10 +104,16 @@ spec: topologyKey: topology.kubernetes.io/zone weight: 1 {{- end }} - {{- with .Values.tolerations }} tolerations: + {{- with .Values.tolerations }} {{- toYaml . | nindent 8 }} {{- end }} + {{- if .Values.useDaemonSet }} + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + {{- end }} + {{- if not .Values.useDaemonSet }} topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname @@ -100,6 +121,7 @@ spec: labelSelector: matchLabels: {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 14 }} + {{- end }} volumes: {{- if .Values.existingConfigSecret }} - name: cloud-config diff --git a/charts/proxmox-cloud-controller-manager/values.yaml b/charts/proxmox-cloud-controller-manager/values.yaml index 5d40f47..80e71e7 100644 --- a/charts/proxmox-cloud-controller-manager/values.yaml +++ b/charts/proxmox-cloud-controller-manager/values.yaml @@ -128,6 +128,11 @@ resources: cpu: 10m memory: 32Mi +# -- Deploy CCM in Daemonset mode. +# CCM will use hostNetwork. +# It allows to use CCM without CNI plugins. +useDaemonSet: false + # -- Deployment update stategy type. # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment updateStrategy: diff --git a/docs/deploy/cloud-controller-manager-daemonset.yml b/docs/deploy/cloud-controller-manager-daemonset.yml new file mode 100644 index 0000000..a68997b --- /dev/null +++ b/docs/deploy/cloud-controller-manager-daemonset.yml @@ -0,0 +1,202 @@ +--- +# Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +--- +# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:proxmox-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:proxmox-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: proxmox-cloud-controller-manager + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: proxmox-cloud-controller-manager + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "v0.4.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + template: + metadata: + annotations: + checksum/config: c69436cb1e16c36ff708b1003d3ca4c6ee6484d2524e2ba7d9b68f473acaa1ca + labels: + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + spec: + enableServiceLinks: false + priorityClassName: system-cluster-critical + serviceAccountName: proxmox-cloud-controller-manager + securityContext: + fsGroup: 10258 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 10258 + runAsNonRoot: true + runAsUser: 10258 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + initContainers: + [] + containers: + - name: proxmox-cloud-controller-manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:v0.4.0" + imagePullPolicy: IfNotPresent + args: + - --v=2 + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers=cloud-node,cloud-node-lifecycle + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + topologyKey: topology.kubernetes.io/zone + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + volumes: + - name: cloud-config + secret: + secretName: proxmox-cloud-controller-manager + defaultMode: 416 diff --git a/docs/deploy/cloud-controller-manager-talos.yml b/docs/deploy/cloud-controller-manager-talos.yml index 885172f..17ad98d 100644 --- a/docs/deploy/cloud-controller-manager-talos.yml +++ b/docs/deploy/cloud-controller-manager-talos.yml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -18,7 +18,7 @@ kind: ClusterRole metadata: name: system:proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -106,7 +106,7 @@ kind: Deployment metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -137,8 +137,6 @@ spec: runAsGroup: 10258 runAsNonRoot: true runAsUser: 10258 - hostAliases: - [] initContainers: [] containers: diff --git a/docs/deploy/cloud-controller-manager.yml b/docs/deploy/cloud-controller-manager.yml index 7b52a1f..1b25727 100644 --- a/docs/deploy/cloud-controller-manager.yml +++ b/docs/deploy/cloud-controller-manager.yml @@ -5,7 +5,7 @@ kind: ServiceAccount metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -18,7 +18,7 @@ kind: ClusterRole metadata: name: system:proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -106,7 +106,7 @@ kind: Deployment metadata: name: proxmox-cloud-controller-manager labels: - helm.sh/chart: proxmox-cloud-controller-manager-0.2.0 + helm.sh/chart: proxmox-cloud-controller-manager-0.2.1 app.kubernetes.io/name: proxmox-cloud-controller-manager app.kubernetes.io/instance: proxmox-cloud-controller-manager app.kubernetes.io/version: "v0.4.0" @@ -137,8 +137,6 @@ spec: runAsGroup: 10258 runAsNonRoot: true runAsUser: 10258 - hostAliases: - [] initContainers: [] containers: