diff --git a/README.md b/README.md index 2aa87b5..b2cd657 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Having a single Kubernetes control plane that spans multiple cloud providers can | [Hetzner](hetzner) | 1.4.0 | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ | | [Openstack](openstack) | 1.3.4 | CCM,CSI | many regions, many zones | ✓ | ✓ | | [Oracle](oracle) | 1.3.4 | CCM,~~CSI~~,Autoscaler | one region, many zones | ✓ | ✓ | -| [Proxmox](proxmox) | 1.3.4 | CCM | one region, one zones | ✓ | ✓ | +| [Proxmox](proxmox) | 1.3.4 | CCM,CSI | one region, mny zones | ✓ | ✓ | | [Scaleway](scaleway) | 1.3.4 | CCM,CSI | one region | ✓ | ✓ | ## Known issues diff --git a/proxmox/.gitignore b/proxmox/.gitignore index 21de6f2..648bbea 100644 --- a/proxmox/.gitignore +++ b/proxmox/.gitignore @@ -1,2 +1,3 @@ _cfgs/ templates/worker.patch.yaml +config.yaml diff --git a/proxmox/Makefile b/proxmox/Makefile index 5ed3fb2..5cd888e 100644 --- a/proxmox/Makefile +++ b/proxmox/Makefile @@ -1,6 +1,10 @@ -ENDPOINT=api.cluster.local CLUSTERNAME:="talos-k8s-proxmox" +CPFIRST := ${shell terraform output -raw controlplane_firstnode 2>/dev/null} +ENDPOINT := ${shell terraform output -raw controlplane_endpoint 2>/dev/null} +ifneq (,$(findstring Warning,${ENDPOINT})) +ENDPOINT := api.cluster.local +endif help: @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) @@ -28,12 +32,12 @@ create-templates: create-controlplane-bootstrap: talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT} - talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 bootstrap + talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} bootstrap create-controlplane: ## Bootstrap first controlplane node terraform apply -auto-approve -target=null_resource.controlplane create-kubeconfig: ## Prepare kubeconfig - talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 kubeconfig . + talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} kubeconfig . kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://${ENDPOINT}:6443 kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system diff --git a/proxmox/instances-controlplane.tf b/proxmox/instances-controlplane.tf index 405a307..c06c1e2 100644 --- a/proxmox/instances-controlplane.tf +++ b/proxmox/instances-controlplane.tf @@ -30,7 +30,8 @@ resource "null_resource" "controlplane_metadata" { content = templatefile("${path.module}/templates/metadata.yaml", { hostname : each.value.name, id : each.value.id, - type : "qemu", + providerID : "proxmox://${var.region}/${each.value.id}", + type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB", zone : each.value.zone, region : var.region, }) @@ -110,13 +111,23 @@ resource "proxmox_vm_qemu" "controlplane" { depends_on = [null_resource.controlplane_metadata] } -resource "local_file" "controlplane" { +resource "local_sensitive_file" "controlplane" { for_each = local.controlplanes content = templatefile("${path.module}/templates/controlplane.yaml.tpl", merge(var.kubernetes, { name = each.value.name ipv4_vip = local.ipv4_vip nodeSubnets = local.controlplane_subnet + clusters = yamlencode({ + clusters = [ + { + token_id = var.proxmox_token_id + token_secret = var.proxmox_token_secret + url = "https://${var.proxmox_host}:8006/api2/json" + region = var.region + }, + ] + }) }) ) filename = "_cfgs/${each.value.name}.yaml" @@ -126,7 +137,7 @@ resource "local_file" "controlplane" { resource "null_resource" "controlplane" { for_each = local.controlplanes provisioner "local-exec" { - command = "sleep 60 && talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml" + command = "echo talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml" } - depends_on = [proxmox_vm_qemu.controlplane, local_file.controlplane] + depends_on = [proxmox_vm_qemu.controlplane, local_sensitive_file.controlplane] } diff --git a/proxmox/instances-web.tf b/proxmox/instances-web.tf index d5f1edc..6e801a1 100644 --- a/proxmox/instances-web.tf +++ b/proxmox/instances-web.tf @@ -57,7 +57,8 @@ resource "null_resource" "web_metadata" { content = templatefile("${path.module}/templates/metadata.yaml", { hostname : each.value.name, id : each.value.id, - type : "qemu", + providerID : "proxmox://${var.region}/${each.value.id}", + type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB", zone : each.value.zone, region : var.region, }) diff --git a/proxmox/instances-worker.tf b/proxmox/instances-worker.tf index 3f4d0a9..79df83d 100644 --- a/proxmox/instances-worker.tf +++ b/proxmox/instances-worker.tf @@ -55,7 +55,8 @@ resource "null_resource" "worker_metadata" { content = templatefile("${path.module}/templates/metadata.yaml", { hostname : each.value.name, id : each.value.id, - type : "qemu", + providerID : "proxmox://${var.region}/${each.value.id}", + type : "${each.value.cpu}VCPU-${floor(each.value.mem / 1024)}GB", zone : each.value.zone, region : var.region, }) diff --git a/proxmox/outputs.tf b/proxmox/outputs.tf index ee306cb..1984412 100644 --- a/proxmox/outputs.tf +++ b/proxmox/outputs.tf @@ -4,6 +4,11 @@ output "controlplane_endpoint" { value = local.ipv4_vip } +output "controlplane_firstnode" { + description = "Kubernetes controlplane first node" + value = try(flatten([for s in local.controlplanes : split("/", s.ipv4)[0]])[0], "127.0.0.1") +} + output "controlplane_apply" { description = "Kubernetes controlplane apply command" value = [for cp in local.controlplanes : diff --git a/proxmox/templates/controlplane.yaml.tpl b/proxmox/templates/controlplane.yaml.tpl index 9286dd5..b29ca6e 100644 --- a/proxmox/templates/controlplane.yaml.tpl +++ b/proxmox/templates/controlplane.yaml.tpl @@ -71,12 +71,23 @@ cluster: - ${nodeSubnets} listenSubnets: - ${nodeSubnets} + inlineManifests: + - name: proxmox-cloud-controller-manager + contents: |- + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: proxmox-cloud-controller-manager + namespace: kube-system + data: + config.yaml: ${base64encode(clusters)} externalCloudProvider: enabled: true manifests: - # - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml - - https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager.yml - # - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin-talos.yml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml + - https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/main/docs/deploy/cloud-controller-manager-talos.yml + - https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/main/docs/deploy/proxmox-csi-plugin-talos.yml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml diff --git a/proxmox/templates/metadata.yaml b/proxmox/templates/metadata.yaml index c308014..420e43a 100644 --- a/proxmox/templates/metadata.yaml +++ b/proxmox/templates/metadata.yaml @@ -1,5 +1,6 @@ hostname: ${hostname} instance-id: ${id} instance-type: ${type} +provider-id: ${providerID} region: ${region} zone: ${zone} diff --git a/proxmox/variables.tf b/proxmox/variables.tf index 9216f61..f7ee43a 100644 --- a/proxmox/variables.tf +++ b/proxmox/variables.tf @@ -38,9 +38,9 @@ variable "proxmox_token_secret" { } variable "region" { - description = "Proxmox host" + description = "Proxmox Cluster Name" type = string - default = "nova" + default = "cluster-1" } variable "kubernetes" {