diff --git a/.dockerignore b/.dockerignore index e18387f..6a4bf79 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,7 +2,10 @@ .github/ .git/ **/.gitignore +# charts/ +docs/ +hack/ Dockerfile /proxmox-cloud-controller-manager* # diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md new file mode 100644 index 0000000..a32ec00 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.md @@ -0,0 +1,11 @@ +--- +name: Feature Requests +about: Create a feature request. +title: "" +labels: "" +assignees: "" +--- + +## Feature Request + +### Description diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..7a590ba --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,23 @@ +# Pull Request + + + +## What? (description) + +## Why? (reasoning) + +## Acceptance + +Please use the following checklist: + +- [ ] you linked an issue (if applicable) +- [ ] you included tests (if applicable) +- [ ] you linted your code (`make lint`) +- [ ] you linted your code (`make unit`) + +> See `make help` for a description of the available targets. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f75dad3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,29 @@ +--- + +# See https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + commit-message: + prefix: "chore:" + open-pull-requests-limit: 5 + rebase-strategy: disabled + schedule: + interval: "weekly" + day: "monday" + time: "07:00" + timezone: "UTC" + + - package-ecosystem: "docker" + directory: "/" + commit-message: + prefix: "chore:" + open-pull-requests-limit: 5 + rebase-strategy: disabled + schedule: + interval: "weekly" + day: "monday" + time: "07:00" + timezone: "UTC" diff --git a/.github/workflows/build-edge.yaml b/.github/workflows/build-edge.yaml new file mode 100644 index 0000000..e2b2c51 --- /dev/null +++ b/.github/workflows/build-edge.yaml @@ -0,0 +1,37 @@ +name: Build edge + +on: + push: + branches: + - main + paths: + - 'go.mod' + - 'go.sum' + - 'cmd/**' + - 'pkg/**' + +jobs: + build-publish: + name: "Build image and publish" + runs-on: ubuntu-22.04 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up docker buildx + run: make docker-init + - name: Github registry login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push + run: make images + env: + PUSH: "true" + TAG: "edge" diff --git a/.github/workflows/build-test.yaml b/.github/workflows/build-test.yaml new file mode 100644 index 0000000..4278bdc --- /dev/null +++ b/.github/workflows/build-test.yaml @@ -0,0 +1,34 @@ +name: Build check + +on: + pull_request: + branches: + - main + paths: + - 'go.mod' + - 'go.sum' + - 'cmd/**' + - 'pkg/**' + +jobs: + build: + name: Build + runs-on: ubuntu-22.04 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up go + uses: actions/setup-go@v3 + with: + go-version-file: 'go.mod' + cache: true + + - name: Build + run: make build + - name: Lint + uses: golangci/golangci-lint-action@v3 + with: + args: --config=.golangci.yml diff --git a/.github/workflows/charts.yaml b/.github/workflows/charts.yaml new file mode 100644 index 0000000..c1ddf34 --- /dev/null +++ b/.github/workflows/charts.yaml @@ -0,0 +1,29 @@ +name: Helm chart check + +on: + pull_request: + branches: + - main + paths: + - 'charts/**' + +jobs: + helm-lint: + name: Helm chart check + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Unshallow + run: git fetch --prune --unshallow + + - name: Install chart-testing tools + id: lint + uses: helm/chart-testing-action@v2.3.1 + + - name: Run helm chart linter + run: ct --config hack/ct.yml lint + - name: Run helm template + run: | + helm template -n kube-system -f charts/talos-cloud-controller-manager/values-tests.yaml \ + ccm charts/talos-cloud-controller-manager > /dev/null diff --git a/.gitignore b/.gitignore index 368fc9b..723b8ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ # +/charts/proxmox-cloud-controller-manager/values-dev.yaml /proxmox-cloud-controller-manager* /kubeconfig +/proxmox-config.yaml # diff --git a/.golangci.yml b/.golangci.yml index d2ecfb5..32d794a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -124,8 +124,7 @@ linters-settings: max-complexity: 20 gomoddirectives: replace-local: true - replace-allow-list: - - cloud.google.com/go + replace-allow-list: [] retract-allow-no-explanation: false exclude-forbidden: true diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4167310 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# syntax = docker/dockerfile:1.4 +######################################## + +FROM --platform=${BUILDPLATFORM} golang:1.20.3-alpine3.17 AS builder +RUN apk update && apk add --no-cache make +ENV GO111MODULE on +WORKDIR /src + +COPY go.mod go.sum /src +RUN go mod download && go mod verify + +COPY . . +ARG TAG +RUN make build-all-archs + +######################################## + +FROM --platform=${TARGETARCH} gcr.io/distroless/static-debian11:nonroot AS release +LABEL org.opencontainers.image.source https://github.com/sergelogvinov/proxmox-cloud-controller-manager + +ARG TARGETARCH +COPY --from=builder /src/proxmox-cloud-controller-manager-${TARGETARCH} /proxmox-cloud-controller-manager + +ENTRYPOINT ["/proxmox-cloud-controller-manager"] diff --git a/Makefile b/Makefile index 4bf3012..3ab039f 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ To build this project, you must have the following installed: - git - make -- golang 1.19 +- golang 1.20+ - golangci-lint endef @@ -45,6 +45,9 @@ help: ## This help menu. # Build Abstractions +build-all-archs: + @for arch in $(ARCHS); do $(MAKE) ARCH=$${arch} build ; done + .PHONY: build build: ## Build CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build $(GO_LDFLAGS) \ @@ -52,7 +55,7 @@ build: ## Build .PHONY: run run: build - ./proxmox-cloud-controller-manager-$(ARCH) --v=4 --kubeconfig=kubeconfig --cloud-config=hack/proxmox-config.yaml --controllers=cloud-node \ + ./proxmox-cloud-controller-manager-$(ARCH) --v=5 --kubeconfig=kubeconfig --cloud-config=proxmox-config.yaml --controllers=cloud-node,cloud-node-lifecycle \ --use-service-account-credentials --leader-elect=false --bind-address=127.0.0.1 .PHONY: lint @@ -62,3 +65,26 @@ lint: ## Lint .PHONY: unit unit: go test -tags=unit $(shell go list ./...) $(TESTARGS) + +.PHONY: docs +docs: + helm template -n kube-system proxmox-cloud-controller-manager \ + --set-string image.tag=$(TAG) \ + charts/proxmox-cloud-controller-manager > docs/deploy/cloud-controller-manager.yml + +# Docker stages + +docker-init: + docker run --rm --privileged multiarch/qemu-user-static:register --reset + + docker context create multiarch ||: + docker buildx create --name multiarch --driver docker-container --use ||: + docker context use multiarch + docker buildx inspect --bootstrap multiarch + +.PHONY: images +images: + @docker buildx build $(BUILD_ARGS) \ + --build-arg TAG=$(TAG) \ + -t $(IMAGE):$(TAG) \ + -f Dockerfile . diff --git a/README.md b/README.md index a1adad0..f3f4fad 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,55 @@ -# proxmox-cloud-controller-manager -Proxmox CCM +# Proxmox Cloud Controller Manager + +## Example + +```yaml +# cloud provider config +clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1 + - url: https://cluster-api-2.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-2 +``` + +Node spec result: + +```yaml +apiVersion: v1 +kind: Node +metadata: + labels: + ... + node.kubernetes.io/instance-type: 2VCPU-2GB + topology.kubernetes.io/region: cluster-1 + topology.kubernetes.io/zone: pve-node-1 + name: worker-1 +spec: + ... + providerID: proxmox://cluster-1/123 +status: + addresses: + - address: 172.16.0.31 + type: InternalIP + - address: worker-1 + type: Hostname +``` + +## Install + +### kubectl + +```shell +kubectl apply -f https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml +``` + +### Helm install + +```shell +helm upgrade -i --namespace=kube-system proxmox-cloud-controller-manager charts/proxmox-cloud-controller-manager +``` diff --git a/charts/proxmox-cloud-controller-manager/.helmignore b/charts/proxmox-cloud-controller-manager/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/proxmox-cloud-controller-manager/Chart.yaml b/charts/proxmox-cloud-controller-manager/Chart.yaml new file mode 100644 index 0000000..d7c564d --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/Chart.yaml @@ -0,0 +1,15 @@ +apiVersion: v2 +name: proxmox-cloud-controller-manager +description: A Helm chart for Kubernetes +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" diff --git a/charts/proxmox-cloud-controller-manager/templates/NOTES.txt b/charts/proxmox-cloud-controller-manager/templates/NOTES.txt new file mode 100644 index 0000000..e69de29 diff --git a/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl b/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl new file mode 100644 index 0000000..ee96292 --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/_helpers.tpl @@ -0,0 +1,69 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "proxmox-cloud-controller-manager.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "proxmox-cloud-controller-manager.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "proxmox-cloud-controller-manager.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "proxmox-cloud-controller-manager.labels" -}} +helm.sh/chart: {{ include "proxmox-cloud-controller-manager.chart" . }} +{{ include "proxmox-cloud-controller-manager.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "proxmox-cloud-controller-manager.selectorLabels" -}} +app.kubernetes.io/name: {{ include "proxmox-cloud-controller-manager.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "proxmox-cloud-controller-manager.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "proxmox-cloud-controller-manager.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Generate string of enabled controllers. Might have a trailing comma (,) which needs to be trimmed. +*/}} +{{- define "proxmox-cloud-controller-manager.enabledControllers" }} +{{- range .Values.enabledControllers -}}{{ . }},{{- end -}} +{{- end }} diff --git a/charts/proxmox-cloud-controller-manager/templates/deployment.yaml b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml new file mode 100644 index 0000000..d48acba --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.updateStrategy.type }} + selector: + matchLabels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "proxmox-cloud-controller-manager.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/proxmox-cloud-controller-manager"] + args: + - --v={{ .Values.logVerbosityLevel }} + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers={{- trimAll "," (include "proxmox-cloud-controller-manager.enabledControllers" . ) }} + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + {{- with .Values.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: cloud-config + secret: + secretName: {{ include "proxmox-cloud-controller-manager.fullname" . }} + defaultMode: 416 # 0640 diff --git a/charts/proxmox-cloud-controller-manager/templates/role.yaml b/charts/proxmox-cloud-controller-manager/templates/role.yaml new file mode 100644 index 0000000..b35bf2b --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/role.yaml @@ -0,0 +1,53 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create diff --git a/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml b/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml new file mode 100644 index 0000000..79d86d3 --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/rolebinding.yaml @@ -0,0 +1,26 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:{{ include "proxmox-cloud-controller-manager.fullname" . }}:extension-apiserver-authentication-reader + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/proxmox-cloud-controller-manager/templates/secrets.yaml b/charts/proxmox-cloud-controller-manager/templates/secrets.yaml new file mode 100644 index 0000000..4a6947c --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/secrets.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "proxmox-cloud-controller-manager.fullname" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: {{ toYaml .Values.config | b64enc | quote }} diff --git a/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml b/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml new file mode 100644 index 0000000..b9ecfcc --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/templates/serviceaccount.yaml @@ -0,0 +1,13 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "proxmox-cloud-controller-manager.serviceAccountName" . }} + labels: + {{- include "proxmox-cloud-controller-manager.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/proxmox-cloud-controller-manager/values-example.yaml b/charts/proxmox-cloud-controller-manager/values-example.yaml new file mode 100644 index 0000000..2d26c81 --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/values-example.yaml @@ -0,0 +1,7 @@ + +image: + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + pullPolicy: Always + tag: edge + +logVerbosityLevel: 4 diff --git a/charts/proxmox-cloud-controller-manager/values.yaml b/charts/proxmox-cloud-controller-manager/values.yaml new file mode 100644 index 0000000..fdcdb0e --- /dev/null +++ b/charts/proxmox-cloud-controller-manager/values.yaml @@ -0,0 +1,115 @@ +# Default values for proxmox-cloud-controller-manager. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/sergelogvinov/proxmox-cloud-controller-manager + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# -- Any extra arguments for talos-cloud-controller-manager +extraArgs: [] + # - --cluster-name=kubernetes + +# -- List of controllers should be enabled. +# Use '*' to enable all controllers. +# Support only `cloud-node,cloud-node-lifecycle` controllers. +enabledControllers: + - cloud-node + - cloud-node-lifecycle + # - route + # - service + +# -- Log verbosity level. See https://github.com/kubernetes/community/blob/master/contributors/devel/sig-instrumentation/logging.md +# for description of individual verbosity levels. +logVerbosityLevel: 2 + +config: + clusters: [] + # - url: https://cluster-api-1.exmple.com:8006/api2/json + # insecure: false + # token_id: "login!name" + # token_secret: "secret" + # region: cluster-1 + +# -- Pods Service Account. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +# -- CCM pods' priorityClassName. +priorityClassName: system-cluster-critical + +# -- Annotations for data pods. +# ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/ +podAnnotations: {} + +# -- Pods Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +podSecurityContext: + runAsNonRoot: true + runAsUser: 10258 + runAsGroup: 10258 + fsGroup: 10258 + fsGroupChangePolicy: "OnRootMismatch" + +# -- Container Security Context. +# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + requests: + cpu: 10m + memory: 32Mi + +# -- Deployment update stategy type. +# ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#updating-a-deployment +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + +# -- Node labels for data pods assignment. +# ref: https://kubernetes.io/docs/user-guide/node-selection/ +nodeSelector: {} + # node-role.kubernetes.io/control-plane: "" + +# -- Tolerations for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + +# -- Affinity for data pods assignment. +# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity +affinity: {} diff --git a/docs/deploy/cloud-controller-manager.yml b/docs/deploy/cloud-controller-manager.yml new file mode 100644 index 0000000..b7c985d --- /dev/null +++ b/docs/deploy/cloud-controller-manager.yml @@ -0,0 +1,198 @@ +--- +# Source: proxmox-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "0.0.1" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "0.0.1" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +data: + config.yaml: "Y2x1c3RlcnM6IFtd" +--- +# Source: proxmox-cloud-controller-manager/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "0.0.1" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +--- +# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:proxmox-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:proxmox-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: proxmox-cloud-controller-manager + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:proxmox-cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: proxmox-cloud-controller-manager + namespace: kube-system +--- +# Source: proxmox-cloud-controller-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxmox-cloud-controller-manager + labels: + helm.sh/chart: proxmox-cloud-controller-manager-0.1.0 + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + app.kubernetes.io/version: "0.0.1" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + template: + metadata: + labels: + app.kubernetes.io/name: proxmox-cloud-controller-manager + app.kubernetes.io/instance: proxmox-cloud-controller-manager + spec: + serviceAccountName: proxmox-cloud-controller-manager + securityContext: + fsGroup: 10258 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 10258 + runAsNonRoot: true + runAsUser: 10258 + containers: + - name: proxmox-cloud-controller-manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/sergelogvinov/proxmox-cloud-controller-manager:811670e" + imagePullPolicy: IfNotPresent + command: ["/proxmox-cloud-controller-manager"] + args: + - --v=2 + - --cloud-provider=proxmox + - --cloud-config=/etc/proxmox/config.yaml + - --controllers=cloud-node,cloud-node-lifecycle + - --leader-elect-resource-name=cloud-controller-manager-proxmox + - --use-service-account-credentials + - --secure-port=10258 + livenessProbe: + httpGet: + path: /healthz + port: 10258 + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 32Mi + volumeMounts: + - name: cloud-config + mountPath: /etc/proxmox + readOnly: true + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Exists + volumes: + - name: cloud-config + secret: + secretName: proxmox-cloud-controller-manager + defaultMode: 416 # 0640 diff --git a/go.mod b/go.mod index be7f79e..e280b56 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/sergelogvinov/proxmox-cloud-controller-manager go 1.20 require ( + github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0 github.com/spf13/pflag v1.0.5 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.3 diff --git a/go.sum b/go.sum index d70f35d..865ccab 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0 h1:RpMkhkY8Vd1fT0CaNxWjw9XVBTmPAErYJfiPxnwFHaM= +github.com/Telmate/proxmox-api-go v0.0.0-20230329163449-4d08b16c14e0/go.mod h1:zQ/B1nkMv6ueUlAEr0D/x5eaFe3rHSScuTc08dcvvPI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/hack/proxmox-config.yaml b/hack/proxmox-config.yaml index e69de29..dfb0f59 100644 --- a/hack/proxmox-config.yaml +++ b/hack/proxmox-config.yaml @@ -0,0 +1,11 @@ +clusters: + - url: https://cluster-api-1.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-1 + - url: https://cluster-api-2.exmple.com:8006/api2/json + insecure: false + token_id: "user!token-id" + token_secret: "secret" + region: cluster-2 diff --git a/pkg/proxmox/client.go b/pkg/proxmox/client.go index 53d2a62..02af83a 100644 --- a/pkg/proxmox/client.go +++ b/pkg/proxmox/client.go @@ -1,18 +1,68 @@ package proxmox import ( - "context" + "crypto/tls" + "os" + + pxapi "github.com/Telmate/proxmox-api-go/proxmox" clientkubernetes "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" ) type client struct { config *cloudConfig + proxmox []pxCluster kclient clientkubernetes.Interface } -func newClient(ctx context.Context, config *cloudConfig) (*client, error) { - return &client{ - config: config, - }, nil +type pxCluster struct { + client *pxapi.Client + region string +} + +func newClient(config *cloudConfig) (*client, error) { + clusters := len(config.Clusters) + if clusters > 0 { + proxmox := make([]pxCluster, clusters) + + for idx, cfg := range config.Clusters { + tlsconf := &tls.Config{InsecureSkipVerify: true} + if !cfg.Insecure { + tlsconf = nil + } + + client, err := pxapi.NewClient(cfg.URL, nil, os.Getenv("PM_HTTP_HEADERS"), tlsconf, "", 600) + if err != nil { + return nil, err + } + + client.SetAPIToken(cfg.TokenID, cfg.TokenSecret) + + if _, err := client.GetVersion(); err != nil { + klog.Errorf("failed to initialized proxmox client in cluster %s: %v", cfg.Region, err) + + return nil, err + } + + proxmox[idx] = pxCluster{client: client, region: cfg.Region} + } + + return &client{ + config: config, + proxmox: proxmox, + }, nil + } + + return nil, nil +} + +func (c *client) GetProxmoxCluster(region string) (*pxCluster, error) { + for _, px := range c.proxmox { + if px.region == region { + return &px, nil + } + } + + return nil, nil } diff --git a/pkg/proxmox/cloud.go b/pkg/proxmox/cloud.go index 2d1a41d..dfeeefd 100644 --- a/pkg/proxmox/cloud.go +++ b/pkg/proxmox/cloud.go @@ -1,3 +1,4 @@ +// Package proxmox is main CCM defenition. package proxmox import ( @@ -37,7 +38,7 @@ func init() { } func newCloud(config *cloudConfig) (cloudprovider.Interface, error) { - client, err := newClient(context.Background(), config) + client, err := newClient(config) if err != nil { return nil, err } @@ -62,6 +63,14 @@ func (c *cloud) Initialize(clientBuilder cloudprovider.ControllerClientBuilder, c.ctx = ctx c.stop = cancel + for _, px := range c.client.proxmox { + if _, err := px.client.GetVersion(); err != nil { + klog.Errorf("failed to initialized proxmox client on region %s: %v", px.region, err) + + return + } + } + // Broadcast the upstream stop signal to all provider-level goroutines // watching the provider's context for cancellation. go func(provider *cloud) { diff --git a/pkg/proxmox/cloud_config.go b/pkg/proxmox/cloud_config.go index 98997e0..cc61694 100644 --- a/pkg/proxmox/cloud_config.go +++ b/pkg/proxmox/cloud_config.go @@ -4,13 +4,16 @@ import ( "io" yaml "gopkg.in/yaml.v3" - - "k8s.io/klog/v2" ) type cloudConfig struct { - Global struct { - } `yaml:"global,omitempty"` + Clusters []struct { + URL string `yaml:"url"` + Insecure bool `yaml:"insecure,omitempty"` + TokenID string `yaml:"token_id,omitempty"` + TokenSecret string `yaml:"token_secret,omitempty"` + Region string `yaml:"region,omitempty"` + } `yaml:"clusters,omitempty"` } func readCloudConfig(config io.Reader) (cloudConfig, error) { @@ -22,7 +25,7 @@ func readCloudConfig(config io.Reader) (cloudConfig, error) { } } - klog.V(4).Infof("cloudConfig: %+v", cfg) + // klog.V(5).Infof("cloudConfig: %+v", cfg) return cfg, nil } diff --git a/pkg/proxmox/instances.go b/pkg/proxmox/instances.go index 0ae2fcf..80beef7 100644 --- a/pkg/proxmox/instances.go +++ b/pkg/proxmox/instances.go @@ -2,9 +2,16 @@ package proxmox import ( "context" + "fmt" + "regexp" + "strconv" + "strings" + + pxapi "github.com/Telmate/proxmox-api-go/proxmox" v1 "k8s.io/api/core/v1" cloudprovider "k8s.io/cloud-provider" + cloudproviderapi "k8s.io/cloud-provider/api" "k8s.io/klog/v2" ) @@ -23,6 +30,23 @@ func newInstances(client *client) *instances { func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, error) { klog.V(4).Info("instances.InstanceExists() called node: ", node.Name) + if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) { + klog.V(4).Infof("instances.InstanceExists() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID) + + return true, nil + } + + _, _, err := i.getInstance(node) + if err != nil { + if err == cloudprovider.InstanceNotFound { + klog.V(4).Infof("instances.InstanceExists() instance %s not found", node.Name) + + return false, nil + } + + return false, err + } + return true, nil } @@ -31,7 +55,32 @@ func (i *instances) InstanceExists(_ context.Context, node *v1.Node) (bool, erro func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, error) { klog.V(4).Info("instances.InstanceShutdown() called, node: ", node.Name) - return true, nil + if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) { + klog.V(4).Infof("instances.InstanceShutdown() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID) + + return false, nil + } + + vmRef, region, err := i.getInstance(node) + if err != nil { + return false, err + } + + px, err := i.c.GetProxmoxCluster(region) + if err != nil { + return false, err + } + + vmState, err := px.client.GetVmState(vmRef) + if err != nil { + return false, err + } + + if vmState["status"].(string) == "stopped" { + return true, nil + } + + return false, nil } // InstanceMetadata returns the instance's metadata. The values returned in InstanceMetadata are @@ -40,5 +89,125 @@ func (i *instances) InstanceShutdown(_ context.Context, node *v1.Node) (bool, er func (i *instances) InstanceMetadata(_ context.Context, node *v1.Node) (*cloudprovider.InstanceMetadata, error) { klog.V(4).Info("instances.InstanceMetadata() called, node: ", node.Name) + if providedIP, ok := node.ObjectMeta.Annotations[cloudproviderapi.AnnotationAlphaProvidedIPAddr]; ok { + var ( + vmRef *pxapi.VmRef + region string + ) + + providerID := node.Spec.ProviderID + if providerID == "" { + klog.V(4).Infof("instances.InstanceMetadata() - trying to find providerID for node %s", node.Name) + + for _, px := range i.c.proxmox { + vm, err := px.client.GetVmRefByName(node.Name) + if err != nil { + continue + } + + vmRef = vm + region = px.region + + break + } + } else if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) { + klog.V(4).Infof("instances.InstanceMetadata() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID) + + return &cloudprovider.InstanceMetadata{}, nil + } + + if vmRef == nil { + var err error + + vmRef, region, err = i.getInstance(node) + if err != nil { + return nil, err + } + } + + addresses := []v1.NodeAddress{{Type: v1.NodeInternalIP, Address: providedIP}} + addresses = append(addresses, v1.NodeAddress{Type: v1.NodeHostName, Address: node.Name}) + + providerID = fmt.Sprintf("%s://%s/%d", ProviderName, region, vmRef.VmId()) + + instanceType, err := i.getInstanceType(vmRef, region) + if err != nil { + instanceType = vmRef.GetVmType() + } + + return &cloudprovider.InstanceMetadata{ + ProviderID: providerID, + NodeAddresses: addresses, + InstanceType: instanceType, + Zone: vmRef.Node(), + Region: region, + }, nil + } + return &cloudprovider.InstanceMetadata{}, nil } + +func (i *instances) getInstance(node *v1.Node) (*pxapi.VmRef, string, error) { + if !strings.HasPrefix(node.Spec.ProviderID, ProviderName) { + klog.V(4).Infof("instances.getInstance() node %s has foreign providerID: %s, skipped", node.Name, node.Spec.ProviderID) + + return nil, "", fmt.Errorf("node %s has foreign providerID: %s", node.Name, node.Spec.ProviderID) + } + + vmid, region, err := i.parseProviderID(node.Spec.ProviderID) + if err != nil { + return nil, "", err + } + + vmRef := pxapi.NewVmRef(vmid) + + px, err := i.c.GetProxmoxCluster(region) + if err != nil { + return nil, "", err + } + + vmInfo, err := px.client.GetVmInfo(vmRef) + if err != nil { + if strings.Contains(err.Error(), "not found") { + return nil, "", cloudprovider.InstanceNotFound + } + + return nil, "", err + } + + klog.V(5).Infof("instances.getInstance() vmInfo %+v", vmInfo) + + return vmRef, region, nil +} + +func (i *instances) getInstanceType(vmRef *pxapi.VmRef, region string) (string, error) { + px, err := i.c.GetProxmoxCluster(region) + if err != nil { + return "", err + } + + vmInfo, err := px.client.GetVmInfo(vmRef) + if err != nil { + return "", err + } + + return fmt.Sprintf("%.0fVCPU-%.0fGB", + vmInfo["maxcpu"].(float64), + vmInfo["maxmem"].(float64)/1024/1024/1024), nil +} + +var providerIDRegexp = regexp.MustCompile(`^` + ProviderName + `://([^/]*)/([^/]+)$`) + +func (i *instances) parseProviderID(providerID string) (int, string, error) { + matches := providerIDRegexp.FindStringSubmatch(providerID) + if len(matches) != 3 { + return 0, "", fmt.Errorf("ProviderID \"%s\" didn't match expected format \"%s://region/InstanceID\"", providerID, ProviderName) + } + + vmID, err := strconv.Atoi(matches[2]) + if err != nil { + return 0, "", err + } + + return vmID, matches[1], nil +}