feat(volumes): provision volumes using tofu

Declaratively provision Proxmox backend persistent volumes for Kubernetes using the Proxmox REST API

remodel/tofu/kubernetes/README.md

@@ -0,0 +1,6 @@
+# Kubernetes Tofu
+
+```shell
+tofu output -raw kube_config
+tofu output -raw talos_config
+```

remodel/tofu/kubernetes/bootstrap/volumes/main.tf

@@ -0,0 +1,30 @@
+module "proxmox-volume" {
+  for_each = var.volumes
+  source  = "./proxmox-volume"
+
+  providers = {
+    restapi = restapi
+  }
+
+  proxmox_api = var.proxmox_api
+  volume = {
+    name = each.key
+    node = each.value.node
+    size = each.value.size
+  }
+}
+
+module "persistent-volume" {
+  for_each = var.volumes
+  source = "./persistent-volume"
+
+  providers = {
+    kubernetes = kubernetes
+  }
+
+  volume = {
+    name          = each.key
+    capacity      = each.value.size
+    volume_handle = "${var.proxmox_api.cluster_name}/${module.proxmox-volume[each.key].node}/${module.proxmox-volume[each.key].storage}/${module.proxmox-volume[each.key].filename}"
+  }
+}

remodel/tofu/kubernetes/bootstrap/volumes/persistent-volume/persistent-volume.tf

@@ -0,0 +1,22 @@
+resource "kubernetes_persistent_volume" "pv" {
+  metadata {
+    name = var.volume.name
+  }
+  spec {
+    capacity = {
+      storage = var.volume.capacity
+    }
+    access_modes       = var.volume.access_modes
+    storage_class_name = var.volume.storage_class_name
+    mount_options      = var.volume.mount_options
+    volume_mode        = var.volume.volume_mode
+    persistent_volume_source {
+      csi {
+        driver            = var.volume.driver
+        fs_type           = var.volume.fs_type
+        volume_handle     = var.volume.volume_handle
+        volume_attributes = var.volume.volume_attributes
+      }
+    }
+  }
+}

remodel/tofu/kubernetes/bootstrap/volumes/persistent-volume/providers.tf

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+      version = ">= 2.31.0"
+    }
+  }
+}

remodel/tofu/kubernetes/bootstrap/volumes/persistent-volume/pv-plex-config.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: plex-config
+spec:
+  storageClassName: proxmox-csi
+  accessModes:
+    - ReadWriteOnce
+  capacity:
+    storage: 12Gi
+  csi:
+    driver: csi.proxmox.sinextra.dev
+    fsType: ext4
+    volumeAttributes:
+      cache: writethrough
+      ssd: "true"
+      storage: local-zfs
+    volumeHandle: homelab/abel/local-zfs/vm-9999-plex-config
+  mountOptions:
+    - noatime
+  volumeMode: Filesystem

remodel/tofu/kubernetes/bootstrap/volumes/persistent-volume/variables.tf

@@ -0,0 +1,19 @@
+variable "volume" {
+  description = "Volume configuration"
+  type = object({
+    name          = string
+    capacity      = string
+    volume_handle = string
+    access_modes = optional(list(string), ["ReadWriteOnce"])
+    storage_class_name = optional(string, "porxmox-csi")
+    fs_type = optional(string, "ext4")
+    driver = optional(string, "csi.proxmox.sinextra.dev")
+    volume_mode = optional(string, "Filesystem")
+    mount_options = optional(list(string), ["noatime"])
+    volume_attributes = optional(object({}), {
+      cache   = "writethrough"
+      ssd     = "true"
+      storage = "local-zfs"
+    })
+  })
+}

remodel/tofu/kubernetes/bootstrap/volumes/providers.tf

@@ -0,0 +1,12 @@
+terraform {
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.31.0"
+    }
+    restapi = {
+      source  = "Mastercard/restapi"
+      version = ">= 1.19.1"
+    }
+  }
+}

remodel/tofu/kubernetes/bootstrap/volumes/proxmox-volume/README.md

@@ -0,0 +1,18 @@
+
+
+```shell
+pvesm alloc local-zfs 8000 vm-8000-app-config 1G
+```
+
+https://pve.proxmox.com/pve-docs/api-viewer/#/nodes/{node}/storage/{storage}/content
+
+```shell
+curl --request POST \
+  --url https://192.168.1.62:8006/api2/json/nodes/abel/storage/local-zfs/content \
+  --header 'Authorization: PVEAPIToken=root@pam!tofu=<UUID>' \
+  --header 'Content-Type: application/x-www-form-urlencoded' \
+  --data vmid=9999 \
+  --data filename=vm-9999-pv-test \
+  --data size=1G \
+  --data format=raw
+```

remodel/tofu/kubernetes/bootstrap/volumes/proxmox-volume/providers.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_providers {
+    restapi = {
+      source  = "Mastercard/restapi"
+      version = ">= 1.19.1"
+    }
+  }
+}
+
+

remodel/tofu/kubernetes/bootstrap/volumes/proxmox-volume/proxmox-volume.tf

@@ -0,0 +1,30 @@
+locals {
+  filename = "vm-${var.volume.vmid}-${var.volume.name}"
+}
+
+resource "restapi_object" "proxmox-volume" {
+  path = "/api2/json/nodes/${var.volume.node}/storage/${var.volume.storage}/content/"
+
+  id_attribute = "data"
+
+  force_new = [var.volume.size]
+
+  data = jsonencode({
+    vmid     = var.volume.vmid
+    filename = local.filename
+    size     = var.volume.size
+    format   = var.volume.format
+  })
+}
+
+output "node" {
+  value = var.volume.node
+}
+
+output "storage" {
+  value = var.volume.storage
+}
+
+output "filename" {
+  value = local.filename
+}

remodel/tofu/kubernetes/bootstrap/volumes/proxmox-volume/variables.tf

@@ -0,0 +1,19 @@
+variable "proxmox_api" {
+  type = object({
+    endpoint  = string
+    insecure  = bool
+    api_token = string
+  })
+  sensitive = true
+}
+
+variable "volume" {
+  type = object({
+    node = string
+    name = string
+    size = string
+    storage = optional(string, "local-zfs")
+    vmid = optional(number, 9999)
+    format = optional(string, "raw")
+  })
+}

remodel/tofu/kubernetes/bootstrap/volumes/variables.tf

@@ -0,0 +1,21 @@
+variable "proxmox_api" {
+  type = object({
+    endpoint     = string
+    insecure     = bool
+    api_token    = string
+    cluster_name = string
+  })
+  sensitive = true
+}
+
+variable "volumes" {
+  type = map(
+    object({
+      node = string
+      size = string
+      storage = optional(string, "local-zfs")
+      vmid = optional(number, 9999)
+      format = optional(string, "raw")
+    })
+  )
+}

remodel/tofu/kubernetes/main.tf

@@ -31,28 +31,20 @@ module "sealed_secrets" {
     kubernetes = kubernetes
   }
 
-  // openssl req -x509 -days 365 -nodes -newkey rsa:4096 -keyout tls.key -out tls.cert -subj "/CN=sealed-secret/O=sealed-secret"
+  // openssl req -x509 -days 365 -nodes -newkey rsa:4096 -keyout sealed-secrets.key -out sealed-secrets.cert -subj "/CN=sealed-secret/O=sealed-secret"
   sealed_secrets_cert = {
-    cert = file("${path.module}/tls.cert")
-    key = file("${path.module}/tls.key")
+    cert = file("${path.module}/bootstrap/sealed-secrets/sealed-secrets.cert")
+    key = file("${path.module}/bootstrap/sealed-secrets/sealed-secrets.key")
   }
 }
 
-resource "local_file" "machine_configs" {
-  for_each        = module.talos.talos_machine_config
-  content         = each.value.machine_configuration
-  filename        = "output/talos-machine-config-${each.key}.yaml"
-  file_permission = "0600"
-}
+module "volumes" {
+  source = "./bootstrap/volumes"
 
-resource "local_file" "talos_config" {
-  content         = module.talos.talos_client_configuration.talos_config
-  filename        = "output/talos-config.yaml"
-  file_permission = "0600"
-}
-
-resource "local_file" "kube_config" {
-  content         = module.talos.talos_kube_config.kubeconfig_raw
-  filename        = "output/kube-config.yaml"
-  file_permission = "0600"
+  providers = {
+    restapi    = restapi
+    kubernetes = kubernetes
+  }
+  proxmox_api = var.proxmox
+  volumes     = var.volumes
 }

remodel/tofu/kubernetes/output.tf

@@ -0,0 +1,28 @@
+resource "local_file" "machine_configs" {
+  for_each        = module.talos.talos_machine_config
+  content         = each.value.machine_configuration
+  filename        = "output/talos-machine-config-${each.key}.yaml"
+  file_permission = "0600"
+}
+
+resource "local_file" "talos_config" {
+  content         = module.talos.talos_client_configuration.talos_config
+  filename        = "output/talos-config.yaml"
+  file_permission = "0600"
+}
+
+resource "local_file" "kube_config" {
+  content         = module.talos.talos_kube_config.kubeconfig_raw
+  filename        = "output/kube-config.yaml"
+  file_permission = "0600"
+}
+
+output "kube_config" {
+  value     = module.talos.talos_kube_config.kubeconfig_raw
+  sensitive = true
+}
+
+output "talos_config" {
+  value     = module.talos.talos_client_configuration.talos_config
+  sensitive = true
+}

remodel/tofu/kubernetes/providers.tf

@@ -12,6 +12,10 @@ terraform {
       source  = "siderolabs/talos"
       version = "0.5.0"
     }
+    restapi = {
+      source  = "Mastercard/restapi"
+      version = "1.19.1"
+    }
   }
 }
 
@@ -26,6 +30,17 @@ provider "proxmox" {
   }
 }
 
+provider "restapi" {
+  uri                  = var.proxmox.endpoint
+  insecure             = var.proxmox.insecure
+  write_returns_object = true
+
+  headers = {
+    "Content-Type"  = "application/json"
+    "Authorization" = "PVEAPIToken=${var.proxmox.api_token}"
+  }
+}
+
 provider "kubernetes" {
   host = module.talos.talos_kube_config.kubernetes_client_configuration.host
   client_certificate = base64decode(module.talos.talos_kube_config.kubernetes_client_configuration.client_certificate)

remodel/tofu/kubernetes/variables.tf

@@ -27,15 +27,29 @@ variable "cluster_config" {
     endpoint        = string
     talos_version   = string
 
-    nodes = map(object({
-      host_node     = string
-      machine_type  = string
-      ip            = string
-      mac_address   = string
-      vm_id         = number
-      cpu           = number
-      ram_dedicated = number
-      igpu = optional(bool, false)
-    }))
+    nodes = map(
+      object({
+        host_node     = string
+        machine_type  = string
+        ip            = string
+        mac_address   = string
+        vm_id         = number
+        cpu           = number
+        ram_dedicated = number
+        igpu = optional(bool, false)
+      })
+    )
   })
 }
+
+variable "volumes" {
+  type = map(
+    object({
+      node = string
+      size = string
+      storage = optional(string, "local-zfs")
+      vmid = optional(number, 9999)
+      format = optional(string, "raw")
+    })
+  )
+}