feat: add controllers

Add cloud-node, cloud-node-lifecycle controllers.

.dockerignore

@@ -2,7 +2,10 @@
 .github/
 .git/
 **/.gitignore
+#
 charts/
+docs/
+hack/
 Dockerfile
 /proxmox-cloud-controller-manager*
 #

.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md

@@ -0,0 +1,11 @@
+---
+name: Feature Requests
+about: Create a feature request.
+title: ""
+labels: ""
+assignees: ""
+---
+
+## Feature Request
+
+### Description

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,23 @@
+# Pull Request
+
+<!--
+## Note to the Contributor
+
+We encourage contributors to go through a proposal process to discuss major changes.
+Before your PR is allowed to run through CI, the maintainers of Talos CCM will first have to approve the PR.
+-->
+
+## What? (description)
+
+## Why? (reasoning)
+
+## Acceptance
+
+Please use the following checklist:
+
+- [ ] you linked an issue (if applicable)
+- [ ] you included tests (if applicable)
+- [ ] you linted your code (`make lint`)
+- [ ] you linted your code (`make unit`)
+
+> See `make help` for a description of the available targets.

.github/dependabot.yml

@@ -0,0 +1,29 @@
+---
+
+# See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    commit-message:
+      prefix: "chore:"
+    open-pull-requests-limit: 5
+    rebase-strategy: disabled
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "07:00"
+      timezone: "UTC"
+
+  - package-ecosystem: "docker"
+    directory: "/"
+    commit-message:
+      prefix: "chore:"
+    open-pull-requests-limit: 5
+    rebase-strategy: disabled
+    schedule:
+      interval: "weekly"
+      day: "monday"
+      time: "07:00"
+      timezone: "UTC"

.github/workflows/build-edge.yaml

@@ -0,0 +1,37 @@
+name: Build edge
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'go.mod'
+      - 'go.sum'
+      - 'cmd/**'
+      - 'pkg/**'
+
+jobs:
+  build-publish:
+    name: "Build image and publish"
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up docker buildx
+        run: make docker-init
+      - name: Github registry login
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        run: make images
+        env:
+          PUSH: "true"
+          TAG: "edge"

.github/workflows/build-test.yaml

@@ -0,0 +1,34 @@
+name: Build check
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'go.mod'
+      - 'go.sum'
+      - 'cmd/**'
+      - 'pkg/**'
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up go
+        uses: actions/setup-go@v3
+        with:
+          go-version-file: 'go.mod'
+          cache: true
+
+      - name: Build
+        run: make build
+      - name: Lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          args: --config=.golangci.yml

.github/workflows/charts.yaml

@@ -0,0 +1,29 @@
+name: Helm chart check
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'charts/**'
+
+jobs:
+  helm-lint:
+    name: Helm chart check
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Unshallow
+        run: git fetch --prune --unshallow
+
+      - name: Install chart-testing tools
+        id: lint
+        uses: helm/chart-testing-action@v2.3.1
+
+      - name: Run helm chart linter
+        run: ct --config hack/ct.yml lint
+      - name: Run helm template
+        run: |
+          helm template -n kube-system -f charts/talos-cloud-controller-manager/values-tests.yaml \
+            ccm charts/talos-cloud-controller-manager > /dev/null

.gitignore

@@ -1,4 +1,6 @@
 #
+/charts/proxmox-cloud-controller-manager/values-dev.yaml
 /proxmox-cloud-controller-manager*
 /kubeconfig
+/proxmox-config.yaml
 #

.golangci.yml

@@ -124,8 +124,7 @@ linters-settings:
     max-complexity: 20
   gomoddirectives:
     replace-local: true
-    replace-allow-list:
+    replace-allow-list: []
-      - cloud.google.com/go
     retract-allow-no-explanation: false
     exclude-forbidden: true
 

Dockerfile

@@ -0,0 +1,24 @@
+# syntax = docker/dockerfile:1.4
+########################################
+
+FROM --platform=${BUILDPLATFORM} golang:1.20.3-alpine3.17 AS builder
+RUN apk update && apk add --no-cache make
+ENV GO111MODULE on
+WORKDIR /src
+
+COPY go.mod go.sum /src
+RUN go mod download && go mod verify
+
+COPY . .
+ARG TAG
+RUN make build-all-archs
+
+########################################
+
+FROM --platform=${TARGETARCH} gcr.io/distroless/static-debian11:nonroot AS release
+LABEL org.opencontainers.image.source https://github.com/sergelogvinov/proxmox-cloud-controller-manager
+
+ARG TARGETARCH
+COPY --from=builder /src/proxmox-cloud-controller-manager-${TARGETARCH} /proxmox-cloud-controller-manager
+
+ENTRYPOINT ["/proxmox-cloud-controller-manager"]

Makefile

@@ -32,7 +32,7 @@ To build this project, you must have the following installed:
 
 - git
 - make
-- golang 1.19
+- golang 1.20+
 - golangci-lint
 
 endef
@@ -45,6 +45,9 @@ help: ## This help menu.
 
 # Build Abstractions
 
+build-all-archs:
+	@for arch in $(ARCHS); do $(MAKE) ARCH=$${arch} build ; done
+
 .PHONY: build
 build: ## Build
 	CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \
@@ -52,7 +55,7 @@ build: ## Build
 
 .PHONY: run
 run: build
-	./proxmox-cloud-controller-manager-$(ARCH) --v=4 --kubeconfig=kubeconfig --cloud-config=hack/proxmox-config.yaml --controllers=cloud-node \
+	./proxmox-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=proxmox-config.yaml --controllers=cloud-node,cloud-node-lifecycle \
 		--use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1
 
 .PHONY: lint
@@ -62,3 +65,26 @@ lint: ## Lint
 .PHONY: unit
 unit:
 	go test -tags=unit $(shell go list ./...) $(TESTARGS)
+
+.PHONY: docs
+docs:
+	helm template -n kube-system proxmox-cloud-controller-manager \
+		--set-string image.tag=$(TAG) \
+		charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager.yml
+
+# Docker stages
+
+docker-init:
+	docker run --rm --privileged multiarch/qemu-user-static:register --reset
+
+	docker context create multiarch ||:
+	docker buildx create --name multiarch --driver docker-container --use ||:
+	docker context use multiarch
+	docker buildx inspect --bootstrap multiarch
+
+.PHONY: images
+images:
+	@docker buildx build $(BUILD_ARGS) \
+		--build-arg TAG=$(TAG) \
+		-t $(IMAGE):$(TAG) \
+		-f Dockerfile .

README.md

@@ -1,2 +1,55 @@
-# proxmox-cloud-controller-manager
+# Proxmox Cloud Controller Manager
-Proxmox CCM
+
+## Example
+
+```yaml
+# cloud provider config
+clusters:
+  - url: https://cluster-api-1.exmple.com:8006/api2/json
+    insecure: false
+    token_id: "user!token-id"
+    token_secret: "secret"
+    region: cluster-1
+  - url: https://cluster-api-2.exmple.com:8006/api2/json
+    insecure: false
+    token_id: "user!token-id"
+    token_secret: "secret"
+    region: cluster-2
+```
+
+Node spec result:
+
+```yaml
+apiVersion: v1
+kind: Node
+metadata:
+  labels:
+    ...
+    node.kubernetes.io/instance-type: 2VCPU-2GB
+    topology.kubernetes.io/region: cluster-1
+    topology.kubernetes.io/zone: pve-node-1
+  name: worker-1
+spec:
+  ...
+  providerID: proxmox://cluster-1/123
+status:
+  addresses:
+  - address: 172.16.0.31
+    type: InternalIP
+  - address: worker-1
+    type: Hostname
+```
+
+## Install
+
+### kubectl
+
+```shell
+kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml
+```
+
+### Helm install
+
+```shell
+helm upgrade -i --namespace=kube-system proxmox-cloud-controller-manager charts/proxmox-cloud-controller-manager
+```

charts/proxmox-cloud-controller-manager/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/proxmox-cloud-controller-manager/Chart.yaml

@@ -0,0 +1,15 @@
+apiVersion: v2
+name: proxmox-cloud-controller-manager
+description: A Helm chart for Kubernetes
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"

charts/proxmox-cloud-controller-manager/templates/_helpers.tpl

@@ -0,0 +1,69 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "proxmox-cloud-controller-manager.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "proxmox-cloud-controller-manager.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "proxmox-cloud-controller-manager.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "proxmox-cloud-controller-manager.labels" -}}
+helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }}
+{{ include "proxmox-cloud-controller-manager.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "proxmox-cloud-controller-manager.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "proxmox-cloud-controller-manager.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
+
+{{/*
+Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed.
+*/}}
+{{- define "proxmox-cloud-controller-manager.enabledControllers" }}
+{{- range .Values.enabledControllers -}}{{ . }},{{- end -}}
+{{- end }}

charts/proxmox-cloud-controller-manager/templates/deployment.yaml

@@ -0,0 +1,79 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "proxmox-cloud-controller-manager.fullname" . }}
+  labels:
+    {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  strategy:
+    type: {{ .Values.updateStrategy.type }}
+  selector:
+    matchLabels:
+      {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command: ["/proxmox-cloud-controller-manager"]
+          args:
+            - --v={{ .Values.logVerbosityLevel }}
+            - --cloud-provider=proxmox
+            - --cloud-config=/etc/proxmox/config.yaml
+            - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }}
+            - --leader-elect-resource-name=cloud-controller-manager-proxmox
+            - --use-service-account-credentials
+            - --secure-port=10258
+          {{- with .Values.extraArgs }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10258
+              scheme: HTTPS
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+            - name: cloud-config
+              mountPath: /etc/proxmox
+              readOnly: true
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: cloud-config
+          secret:
+            secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }}
+            defaultMode: 416 # 0640

charts/proxmox-cloud-controller-manager/templates/role.yaml

@@ -0,0 +1,53 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}
+  labels:
+    {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create

charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml

@@ -0,0 +1,26 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "proxmox-cloud-controller-manager.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "proxmox-cloud-controller-manager.fullname" . }}
+    namespace: {{ .Release.Namespace }}

charts/proxmox-cloud-controller-manager/templates/secrets.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "proxmox-cloud-controller-manager.fullname" . }}
+  labels:
+    {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }}
+  namespace: {{ .Release.Namespace }}
+data:
+  config.yaml: {{ toYaml .Values.config | b64enc | quote }}

charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }}
+  labels:
+    {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

charts/proxmox-cloud-controller-manager/values-example.yaml

@@ -0,0 +1,7 @@
+
+image:
+  repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager
+  pullPolicy: Always
+  tag: edge
+
+logVerbosityLevel: 4

charts/proxmox-cloud-controller-manager/values.yaml

@@ -0,0 +1,115 @@
+# Default values for proxmox-cloud-controller-manager.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+# -- Any extra arguments for talos-cloud-controller-manager
+extraArgs: []
+  # - --cluster-name=kubernetes
+
+# -- List of controllers should be enabled.
+# Use '*' to enable all controllers.
+# Support only `cloud-node,cloud-node-lifecycle` controllers.
+enabledControllers:
+  - cloud-node
+  - cloud-node-lifecycle
+  # - route
+  # - service
+
+# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md
+# for description of individual verbosity levels.
+logVerbosityLevel: 2
+
+config:
+  clusters: []
+  #   - url: https://cluster-api-1.exmple.com:8006/api2/json
+  #     insecure: false
+  #     token_id: "login!name"
+  #     token_secret: "secret"
+  #     region: cluster-1
+
+# -- Pods Service Account.
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+# -- CCM pods' priorityClassName.
+priorityClassName: system-cluster-critical
+
+# -- Annotations for data pods.
+# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+podAnnotations: {}
+
+# -- Pods Security Context.
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+podSecurityContext:
+  runAsNonRoot: true
+  runAsUser: 10258
+  runAsGroup: 10258
+  fsGroup: 10258
+  fsGroupChangePolicy: "OnRootMismatch"
+
+# -- Container Security Context.
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - ALL
+  seccompProfile:
+    type: RuntimeDefault
+
+resources:
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  requests:
+    cpu: 10m
+    memory: 32Mi
+
+# -- Deployment update stategy type.
+# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment
+updateStrategy:
+  type: RollingUpdate
+  rollingUpdate:
+    maxUnavailable: 1
+
+# -- Node labels for data pods assignment.
+# ref: https://kubernetes.io/docs/user-guide/node-selection/
+nodeSelector: {}
+  # node-role.kubernetes.io/control-plane: ""
+
+# -- Tolerations for data pods assignment.
+# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+tolerations:
+  - effect: NoSchedule
+    key: node-role.kubernetes.io/control-plane
+    operator: Exists
+  - effect: NoSchedule
+    key: node.cloudprovider.kubernetes.io/uninitialized
+    operator: Exists
+
+# -- Affinity for data pods assignment.
+# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+affinity: {}

docs/deploy/cloud-controller-manager.yml

@@ -0,0 +1,198 @@
+---
+# Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: proxmox-cloud-controller-manager
+  labels:
+    helm.sh/chart: proxmox-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: proxmox-cloud-controller-manager
+    app.kubernetes.io/instance: proxmox-cloud-controller-manager
+    app.kubernetes.io/version: "0.0.1"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+---
+# Source: proxmox-cloud-controller-manager/templates/secrets.yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: proxmox-cloud-controller-manager
+  labels:
+    helm.sh/chart: proxmox-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: proxmox-cloud-controller-manager
+    app.kubernetes.io/instance: proxmox-cloud-controller-manager
+    app.kubernetes.io/version: "0.0.1"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+data:
+  config.yaml: "Y2x1c3RlcnM6IFtd"
+---
+# Source: proxmox-cloud-controller-manager/templates/role.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:proxmox-cloud-controller-manager
+  labels:
+    helm.sh/chart: proxmox-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: proxmox-cloud-controller-manager
+    app.kubernetes.io/instance: proxmox-cloud-controller-manager
+    app.kubernetes.io/version: "0.0.1"
+    app.kubernetes.io/managed-by: Helm
+rules:
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - create
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - create
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts/token
+  verbs:
+  - create
+---
+# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:proxmox-cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:proxmox-cloud-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: proxmox-cloud-controller-manager
+  namespace: kube-system
+---
+# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+  - kind: ServiceAccount
+    name: proxmox-cloud-controller-manager
+    namespace: kube-system
+---
+# Source: proxmox-cloud-controller-manager/templates/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: proxmox-cloud-controller-manager
+  labels:
+    helm.sh/chart: proxmox-cloud-controller-manager-0.1.0
+    app.kubernetes.io/name: proxmox-cloud-controller-manager
+    app.kubernetes.io/instance: proxmox-cloud-controller-manager
+    app.kubernetes.io/version: "0.0.1"
+    app.kubernetes.io/managed-by: Helm
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: proxmox-cloud-controller-manager
+      app.kubernetes.io/instance: proxmox-cloud-controller-manager
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: proxmox-cloud-controller-manager
+        app.kubernetes.io/instance: proxmox-cloud-controller-manager
+    spec:
+      serviceAccountName: proxmox-cloud-controller-manager
+      securityContext:
+        fsGroup: 10258
+        fsGroupChangePolicy: OnRootMismatch
+        runAsGroup: 10258
+        runAsNonRoot: true
+        runAsUser: 10258
+      containers:
+        - name: proxmox-cloud-controller-manager
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            seccompProfile:
+              type: RuntimeDefault
+          image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:811670e"
+          imagePullPolicy: IfNotPresent
+          command: ["/proxmox-cloud-controller-manager"]
+          args:
+            - --v=2
+            - --cloud-provider=proxmox
+            - --cloud-config=/etc/proxmox/config.yaml
+            - --controllers=cloud-node,cloud-node-lifecycle
+            - --leader-elect-resource-name=cloud-controller-manager-proxmox
+            - --use-service-account-credentials
+            - --secure-port=10258
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 10258
+              scheme: HTTPS
+            initialDelaySeconds: 20
+            periodSeconds: 30
+            timeoutSeconds: 5
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+          volumeMounts:
+            - name: cloud-config
+              mountPath: /etc/proxmox
+              readOnly: true
+      tolerations:
+        - effect: NoSchedule
+          key: node-role.kubernetes.io/control-plane
+          operator: Exists
+        - effect: NoSchedule
+          key: node.cloudprovider.kubernetes.io/uninitialized
+          operator: Exists
+      volumes:
+        - name: cloud-config
+          secret:
+            secretName: proxmox-cloud-controller-manager
+            defaultMode: 416 # 0640

go.mod

@@ -3,6 +3,7 @@ module github.com/sergelogvinov/proxmox-cloud-controller-manager
 go 1.20
 
 require (
+	github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.26.3

go.sum

@@ -42,6 +42,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0 h1:RpMkhkY8Vd1fT0CaNxWjw9XVBTmPAErYJfiPxnwFHaM=
+github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0/go.mod h1:zQ/B1nkMv6ueUlAEr0D/x5eaFe3rHSScuTc08dcvvPI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

hack/proxmox-config.yaml

@@ -0,0 +1,11 @@
+clusters:
+  - url: https://cluster-api-1.exmple.com:8006/api2/json
+    insecure: false
+    token_id: "user!token-id"
+    token_secret: "secret"
+    region: cluster-1
+  - url: https://cluster-api-2.exmple.com:8006/api2/json
+    insecure: false
+    token_id: "user!token-id"
+    token_secret: "secret"
+    region: cluster-2

pkg/proxmox/client.go

@@ -1,18 +1,68 @@
 package proxmox
 
 import (
-	"context"
+	"crypto/tls"
+	"os"
+
+	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
 
 	clientkubernetes "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 )
 
 type client struct {
 	config  *cloudConfig
+	proxmox []pxCluster
 	kclient clientkubernetes.Interface
 }
 
-func newClient(ctx context.Context, config *cloudConfig) (*client, error) {
+type pxCluster struct {
-	return &client{
+	client *pxapi.Client
-		config: config,
+	region string
-	}, nil
+}
+
+func newClient(config *cloudConfig) (*client, error) {
+	clusters := len(config.Clusters)
+	if clusters > 0 {
+		proxmox := make([]pxCluster, clusters)
+
+		for idx, cfg := range config.Clusters {
+			tlsconf := &tls.Config{InsecureSkipVerify: true}
+			if !cfg.Insecure {
+				tlsconf = nil
+			}
+
+			client, err := pxapi.NewClient(cfg.URL, nil, os.Getenv("PM_HTTP_HEADERS"), tlsconf, "", 600)
+			if err != nil {
+				return nil, err
+			}
+
+			client.SetAPIToken(cfg.TokenID, cfg.TokenSecret)
+
+			if _, err := client.GetVersion(); err != nil {
+				klog.Errorf("failed to initialized proxmox client in cluster %s: %v", cfg.Region, err)
+
+				return nil, err
+			}
+
+			proxmox[idx] = pxCluster{client: client, region: cfg.Region}
+		}
+
+		return &client{
+			config:  config,
+			proxmox: proxmox,
+		}, nil
+	}
+
+	return nil, nil
+}
+
+func (c *client) GetProxmoxCluster(region string) (*pxCluster, error) {
+	for _, px := range c.proxmox {
+		if px.region == region {
+			return &px, nil
+		}
+	}
+
+	return nil, nil
 }

pkg/proxmox/cloud.go

@@ -1,3 +1,4 @@
+// Package proxmox is main CCM defenition.
 package proxmox
 
 import (
@@ -37,7 +38,7 @@ func init() {
 }
 
 func newCloud(config *cloudConfig) (cloudprovider.Interface, error) {
-	client, err := newClient(context.Background(), config)
+	client, err := newClient(config)
 	if err != nil {
 		return nil, err
 	}
@@ -62,6 +63,14 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder,
 	c.ctx = ctx
 	c.stop = cancel
 
+	for _, px := range c.client.proxmox {
+		if _, err := px.client.GetVersion(); err != nil {
+			klog.Errorf("failed to initialized proxmox client on region %s: %v", px.region, err)
+
+			return
+		}
+	}
+
 	// Broadcast the upstream stop signal to all provider-level goroutines
 	// watching the provider's context for cancellation.
 	go func(provider *cloud) {

pkg/proxmox/cloud_config.go

@@ -4,13 +4,16 @@ import (
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
-
-	"k8s.io/klog/v2"
 )
 
 type cloudConfig struct {
-	Global struct {
+	Clusters []struct {
-	} `yaml:"global,omitempty"`
+		URL         string `yaml:"url"`
+		Insecure    bool   `yaml:"insecure,omitempty"`
+		TokenID     string `yaml:"token_id,omitempty"`
+		TokenSecret string `yaml:"token_secret,omitempty"`
+		Region      string `yaml:"region,omitempty"`
+	} `yaml:"clusters,omitempty"`
 }
 
 func readCloudConfig(config io.Reader) (cloudConfig, error) {
@@ -22,7 +25,7 @@ func readCloudConfig(config io.Reader) (cloudConfig, error) {
 		}
 	}
 
-	klog.V(4).Infof("cloudConfig: %+v", cfg)
+	// klog.V(5).Infof("cloudConfig: %+v", cfg)
 
 	return cfg, nil
 }

pkg/proxmox/instances.go

@@ -2,9 +2,16 @@ package proxmox
 
 import (
 	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	pxapi "github.com/Telmate/proxmox-api-go/proxmox"
 
 	v1 "k8s.io/api/core/v1"
 	cloudprovider "k8s.io/cloud-provider"
+	cloudproviderapi "k8s.io/cloud-provider/api"
 	"k8s.io/klog/v2"
 )
 
@@ -23,6 +30,23 @@ func newInstances(client *client) *instances {
 func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, error) {
 	klog.V(4).Info("instances.InstanceExists() called node: ", node.Name)
 
+	if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) {
+		klog.V(4).Infof("instances.InstanceExists() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID)
+
+		return true, nil
+	}
+
+	_, _, err := i.getInstance(node)
+	if err != nil {
+		if err == cloudprovider.InstanceNotFound {
+			klog.V(4).Infof("instances.InstanceExists() instance %s not found", node.Name)
+
+			return false, nil
+		}
+
+		return false, err
+	}
+
 	return true, nil
 }
 
@@ -31,7 +55,32 @@ func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, erro
 func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, error) {
 	klog.V(4).Info("instances.InstanceShutdown() called, node: ", node.Name)
 
-	return true, nil
+	if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) {
+		klog.V(4).Infof("instances.InstanceShutdown() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID)
+
+		return false, nil
+	}
+
+	vmRef, region, err := i.getInstance(node)
+	if err != nil {
+		return false, err
+	}
+
+	px, err := i.c.GetProxmoxCluster(region)
+	if err != nil {
+		return false, err
+	}
+
+	vmState, err := px.client.GetVmState(vmRef)
+	if err != nil {
+		return false, err
+	}
+
+	if vmState["status"].(string) == "stopped" {
+		return true, nil
+	}
+
+	return false, nil
 }
 
 // InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are
@@ -40,5 +89,125 @@ func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, er
 func (i *instances) InstanceMetadata(_ context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) {
 	klog.V(4).Info("instances.InstanceMetadata() called, node: ", node.Name)
 
+	if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok {
+		var (
+			vmRef  *pxapi.VmRef
+			region string
+		)
+
+		providerID := node.Spec.ProviderID
+		if providerID == "" {
+			klog.V(4).Infof("instances.InstanceMetadata() - trying to find providerID for node %s", node.Name)
+
+			for _, px := range i.c.proxmox {
+				vm, err := px.client.GetVmRefByName(node.Name)
+				if err != nil {
+					continue
+				}
+
+				vmRef = vm
+				region = px.region
+
+				break
+			}
+		} else if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) {
+			klog.V(4).Infof("instances.InstanceMetadata() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID)
+
+			return &cloudprovider.InstanceMetadata{}, nil
+		}
+
+		if vmRef == nil {
+			var err error
+
+			vmRef, region, err = i.getInstance(node)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: providedIP}}
+		addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name})
+
+		providerID = fmt.Sprintf("%s://%s/%d", ProviderName, region, vmRef.VmId())
+
+		instanceType, err := i.getInstanceType(vmRef, region)
+		if err != nil {
+			instanceType = vmRef.GetVmType()
+		}
+
+		return &cloudprovider.InstanceMetadata{
+			ProviderID:    providerID,
+			NodeAddresses: addresses,
+			InstanceType:  instanceType,
+			Zone:          vmRef.Node(),
+			Region:        region,
+		}, nil
+	}
+
 	return &cloudprovider.InstanceMetadata{}, nil
 }
+
+func (i *instances) getInstance(node *v1.Node) (*pxapi.VmRef, string, error) {
+	if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) {
+		klog.V(4).Infof("instances.getInstance() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID)
+
+		return nil, "", fmt.Errorf("node %s has foreign providerID: %s", node.Name, node.Spec.ProviderID)
+	}
+
+	vmid, region, err := i.parseProviderID(node.Spec.ProviderID)
+	if err != nil {
+		return nil, "", err
+	}
+
+	vmRef := pxapi.NewVmRef(vmid)
+
+	px, err := i.c.GetProxmoxCluster(region)
+	if err != nil {
+		return nil, "", err
+	}
+
+	vmInfo, err := px.client.GetVmInfo(vmRef)
+	if err != nil {
+		if strings.Contains(err.Error(), "not found") {
+			return nil, "", cloudprovider.InstanceNotFound
+		}
+
+		return nil, "", err
+	}
+
+	klog.V(5).Infof("instances.getInstance() vmInfo %+v", vmInfo)
+
+	return vmRef, region, nil
+}
+
+func (i *instances) getInstanceType(vmRef *pxapi.VmRef, region string) (string, error) {
+	px, err := i.c.GetProxmoxCluster(region)
+	if err != nil {
+		return "", err
+	}
+
+	vmInfo, err := px.client.GetVmInfo(vmRef)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%.0fVCPU-%.0fGB",
+		vmInfo["maxcpu"].(float64),
+		vmInfo["maxmem"].(float64)/1024/1024/1024), nil
+}
+
+var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `://([^/]*)/([^/]+)$`)
+
+func (i *instances) parseProviderID(providerID string) (int, string, error) {
+	matches := providerIDRegexp.FindStringSubmatch(providerID)
+	if len(matches) != 3 {
+		return 0, "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"%s://region/InstanceID\"", providerID, ProviderName)
+	}
+
+	vmID, err := strconv.Atoi(matches[2])
+	if err != nil {
+		return 0, "", err
+	}
+
+	return vmID, matches[1], nil
+}