update proxmox setup

README.md

@@ -28,7 +28,7 @@ Having a single Kubernetes control plane that spans multiple cloud providers can
 | [Hetzner](hetzner)     | 1.4.0  | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ |
 | [Openstack](openstack) | 1.3.4  | CCM,CSI            | many regions, many zones | ✓ | ✓ |
 | [Oracle](oracle)       | 1.3.4  | CCM,~~CSI~~,Autoscaler | one region, many zones | ✓ | ✓ |
-| [Proxmox](proxmox)     | 1.3.4  | CCM                | one region, one zones | ✓ | ✓ |
+| [Proxmox](proxmox)     | 1.3.4  | CCM,CSI            | one region, mny zones | ✓ | ✓ |
 | [Scaleway](scaleway)   | 1.3.4  | CCM,CSI            | one region | ✓ | ✓ |
 
 ## Known issues

proxmox/.gitignore

@@ -1,2 +1,3 @@
 _cfgs/
 templates/worker.patch.yaml
+config.yaml

proxmox/Makefile

@@ -1,6 +1,10 @@
 
-ENDPOINT=api.cluster.local
 CLUSTERNAME:="talos-k8s-proxmox"
+CPFIRST := ${shell terraform output -raw controlplane_firstnode 2>/dev/null}
+ENDPOINT := ${shell terraform output -raw controlplane_endpoint 2>/dev/null}
+ifneq (,$(findstring Warning,${ENDPOINT}))
+ENDPOINT := api.cluster.local
+endif
 
 help:
 	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -28,12 +32,12 @@ create-templates:
 
 create-controlplane-bootstrap:
 	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
-	talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 bootstrap
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} bootstrap
 
 create-controlplane: ## Bootstrap first controlplane node
 	terraform apply -auto-approve -target=null_resource.controlplane
 
 create-kubeconfig: ## Prepare kubeconfig
-	talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 kubeconfig .
+	talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} kubeconfig .
 	kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://${ENDPOINT}:6443
 	kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system

proxmox/instances-controlplane.tf

@@ -30,7 +30,8 @@ resource "null_resource" "controlplane_metadata" {
     content = templatefile("${path.module}/templates/metadata.yaml", {
       hostname : each.value.name,
       id : each.value.id,
-      type : "qemu",
+      providerID : "proxmox://${var.region}/${each.value.id}",
+      type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB",
       zone : each.value.zone,
       region : var.region,
     })
@@ -110,13 +111,23 @@ resource "proxmox_vm_qemu" "controlplane" {
   depends_on = [null_resource.controlplane_metadata]
 }
 
-resource "local_file" "controlplane" {
+resource "local_sensitive_file" "controlplane" {
   for_each = local.controlplanes
   content = templatefile("${path.module}/templates/controlplane.yaml.tpl",
     merge(var.kubernetes, {
       name        = each.value.name
       ipv4_vip    = local.ipv4_vip
       nodeSubnets = local.controlplane_subnet
+      clusters = yamlencode({
+        clusters = [
+          {
+            token_id     = var.proxmox_token_id
+            token_secret = var.proxmox_token_secret
+            url          = "https://${var.proxmox_host}:8006/api2/json"
+            region       = var.region
+          },
+        ]
+      })
     })
   )
   filename        = "_cfgs/${each.value.name}.yaml"
@@ -126,7 +137,7 @@ resource "local_file" "controlplane" {
 resource "null_resource" "controlplane" {
   for_each = local.controlplanes
   provisioner "local-exec" {
-    command = "sleep 60 && talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml"
+    command = "echo talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml"
   }
-  depends_on = [proxmox_vm_qemu.controlplane, local_file.controlplane]
+  depends_on = [proxmox_vm_qemu.controlplane, local_sensitive_file.controlplane]
 }

proxmox/instances-web.tf

@@ -57,7 +57,8 @@ resource "null_resource" "web_metadata" {
     content = templatefile("${path.module}/templates/metadata.yaml", {
       hostname : each.value.name,
       id : each.value.id,
-      type : "qemu",
+      providerID : "proxmox://${var.region}/${each.value.id}",
+      type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB",
       zone : each.value.zone,
       region : var.region,
     })

proxmox/instances-worker.tf

@@ -55,7 +55,8 @@ resource "null_resource" "worker_metadata" {
     content = templatefile("${path.module}/templates/metadata.yaml", {
       hostname : each.value.name,
       id : each.value.id,
-      type : "qemu",
+      providerID : "proxmox://${var.region}/${each.value.id}",
+      type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB",
       zone : each.value.zone,
       region : var.region,
     })

proxmox/outputs.tf

@@ -4,6 +4,11 @@ output "controlplane_endpoint" {
   value       = local.ipv4_vip
 }
 
+output "controlplane_firstnode" {
+  description = "Kubernetes controlplane first node"
+  value       = try(flatten([for s in local.controlplanes : split("/", s.ipv4)[0]])[0], "127.0.0.1")
+}
+
 output "controlplane_apply" {
   description = "Kubernetes controlplane apply command"
   value = [for cp in local.controlplanes :

proxmox/templates/controlplane.yaml.tpl

@@ -71,12 +71,23 @@ cluster:
       - ${nodeSubnets}
     listenSubnets:
       - ${nodeSubnets}
+  inlineManifests:
+    - name: proxmox-cloud-controller-manager
+      contents: |-
+        apiVersion: v1
+        kind: Secret
+        type: Opaque
+        metadata:
+          name: proxmox-cloud-controller-manager
+          namespace: kube-system
+        data:
+          config.yaml: ${base64encode(clusters)}
   externalCloudProvider:
     enabled: true
     manifests:
-      # - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml
-      - https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml
+      - https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager-talos.yml
-      # - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin-talos.yml
+      - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin-talos.yml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml
       - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml

proxmox/templates/metadata.yaml

@@ -1,5 +1,6 @@
 hostname: ${hostname}
 instance-id: ${id}
 instance-type: ${type}
+provider-id: ${providerID}
 region: ${region}
 zone: ${zone}

proxmox/variables.tf

@@ -38,9 +38,9 @@ variable "proxmox_token_secret" {
 }
 
 variable "region" {
-  description = "Proxmox host"
+  description = "Proxmox Cluster Name"
   type        = string
-  default     = "nova"
+  default     = "cluster-1"
 }
 
 variable "kubernetes" {