feat(talos): generate schematic id on-the-fly and add possibility of changing image

Utilize Talos Factory API to generate schematic ID which can be used to download images

Add possibility of running two different images in cluster. This allows changing version or schematic for the cluster in place

remodel/k8s/README.md

@@ -1,5 +1,13 @@
 # Manual bootstrap
 
+## CRDs
+
+Gateway API
+
+```shell
+kubectl apply -f https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.0.0/experimental-install.yaml
+```
+
 ## Cilium
 
 ```shell
@@ -15,7 +23,7 @@ kustomize build --enable-helm infra/controllers/sealed-secrets | kubectl apply -
 ## Proxmox CSI Plugin
 
 ```shell
-kubectl kustomize --enable-helm infra/storage/proxmox-csi | kubectl apply -f -
+kustomize build --enable-helm infra/storage/proxmox-csi | kubectl apply -f -
 ```
 
 ```shell
@@ -25,15 +33,7 @@ kubectl get csistoragecapacities -ocustom-columns=CLASS:.storageClassName,AVAIL:
 ## Argo CD
 
 ```shell
-kubeseal -oyaml --controller-namespace=sealed-secrets < argocd-docker-secret.yaml > infra/argocd/docker-helm-credentials.yaml
+kustomize build --enable-helm infra/controllers/argocd | kubectl apply -f -
-```
-
-```shell
-kubeseal -oyaml --controller-namespace=sealed-secrets < argocd-ghcr-secret.yaml > infra/argocd/ghcr-helm-credentials.yaml
-```
-
-```shell
-kustomize build --enable-helm infra/argocd | kubectl apply -f -
 ```
 
 ```shell
@@ -41,18 +41,9 @@ kubectl -n argocd get secret argocd-initial-admin-secret -ojson | jq -r ' .data.
 ```
 
 ```shell
-kubectl kustomize --enable-helm infra/storage | kubectl apply -f -
+kubectl kustomize infra | kubectl apply -f -
 ```
 
-```shell
-kubectl kustomize --enable-helm infra/controllers | kubectl apply -f -
-```
-
-```shell
-kubectl kustomize --enable-helm infra | kubectl apply -f -
-```
-
-
 # SBOM
 
 * [x] Cilium

remodel/k8s/infra/controllers/argocd/docker-helm-credentials.yaml

@@ -1,13 +0,0 @@
-apiVersion: bitnami.com/v1alpha1
-kind: SealedSecret
-metadata:
-  name: argocd-docker-helm-credentials
-  namespace: argocd
-spec:
-  encryptedData:
-    password: AgDYFcrvLka10TTS5ozL/1o5VZDe7eRsYNHKAwx5yn+/taEtCP5a2ldzOdFYWdrdCPXQHdJk0vBnf2qKzyTem2HiWy42vqtOe95KujPT/WD4mnoAwU9iI4aQQjfI4tj4BJuo2hyCA4kS9V5PPmNul3wa6vigbrnPBgYPMT3vlMpIW/eKWcGvX/uvX8JWdg1LYSQqIhW5d8pzTu6cVoWyJwv1Sh5yfy33sbjQ5Mdy1LQEV4O5J+YjCDTIuEYtJo7GFuhxMseJIwy8m0NmiAj/ItG9H3n8TVj0SBlICNPCL3Du9oXCgHy3IfPymhoC9aWLgnMA9vhn0h96U2hbzRzLufVkogM869nSqfty45xcAPxRCtPV6Yr5JlGRE0CmlLCw7NIbTBU2CREcBARQLNbO5Rrp0bUvLOuP9CxLwppHkZJmNbJ7UYFh5q/5a0tT3cBp9PjC4PoMsoEg9EydmSd4layxIGRcy/tkdbYqyzheQNWV1Vv5iJ0ecArnt/jjMyAxwRYkeNoTVb8g6QSjFxtUErbXHNK5YJKywLFDrrus/QTNZE2q/xDRWpPKJpF74eCNFwCm2Mk3BXwvTfGgze0szuu1nwzkgYnozmGAZCmwlG5UQgFMrgB1RGABScYEiXtqowfH0bSjQHAAv5E6XnDAhO1qZEl205ohnGhza30QAXUlVNHNCNnvJkS5XL/+hAsY9Afrda9bVh3xhLnk0ckYF4+vni24NaIzqMecCcmMV/RHbcirfpI=
-    username: AgAPHJn1UFPef0ZEbeGQ8wZpfIPMJ1UWLyqkIaXFtGJYDO82OsU6pwxzJpKYWXjysSg3S0RNraA3JsObs5NOcxB5dOecKNs50k5s/mAGrS5n/q2LRAekRZFYRYNBQhaa17Q2D7vVkj2gYNhCJkk9Orgc1N7iJbpienNEH/R0bjplGa2AvOk5T4HB8qaQB5LTLR7it3gVzak/Uo3fl3TATGF/YBgHVWsZioEIxs7Z8QK9R2NCWQm+63ddXKqvnZjqDf2KTKYTLlNcmEi8XNjW2/5CfWfZbB/7d5jbqBYOSLsr0SZ4JuAOeFhifOVqIXqHm0CBV9jB0IoL7Nlxoy4jD9SUN2CackAuktM+LDs8BEeKiB0YAMIiX/4pkaxGnc8D1n/6AfRZ/tlJgIhLkB6BjjsYhObfCKQRTu8pGUpOyoHrCalZW6Cx3pz5Qjcidac8K4CN9kCuIGUlpwXUi1jgPioJRfPRwlGdQpZK51vE4FiNByDntCSr4vSDcGNxuE4kwwPJz0pXhZG+pRnGVYe4ok+bHgyxlw/OzOqqj+UdD6Sv1FoLvyibxIYj0ZmsGCbxwI9wZR6Wmx1w9OV2H25k4QDpH8yXOtCag6PPF7Y9LTN6aeX3CrXV3bMBUE4NSia7WQ+sDs0vGQlmz9a/yUTw6ziD5XIqqOHUF1XJy2Hk6JZ6MJRKzf0xfEY2b2fF2qLl1H2Zl9dbL60=
-  template:
-    metadata:
-      name: argocd-docker-helm-credentials
-      namespace: argocd

remodel/k8s/infra/controllers/argocd/ghcr-helm-credentials.yaml

@@ -1,13 +0,0 @@
-apiVersion: bitnami.com/v1alpha1
-kind: SealedSecret
-metadata:
-  name: argocd-ghcr-helm-credentials
-  namespace: argocd
-spec:
-  encryptedData:
-    password: AgDJl0Iry4mvGquzKgJ+2cthSMbbkDigY0t9D4uy8cL7cq4TFCFgy/yPsBnjbFjbRPMCo0HeaTPcvsQn0oidex8KfCccRZZJhj7U9i5p0yL5GhEWFgqejh0/82GviXtz9apDdkeqHCAxO7SP9MlrPKPQb1Je2RcSWl5sqRbGcCRxAZ5655Okx8k3ReFOFCBSqu22asQ/SHTOHHJgdT1tZ2jIV4vYtbOSWIRToSsXOxiHHqczrjTmj/LhpH4dKHUNowZxlV9V6Sklr0iXsOZQ4uWMraO14iWEMcX4TLbpXUr4ynLuqHNraoLvSd8/qiJOjcj+TSAFMZ0gkKJo1oLkvHWfVCh0oTAtAztlOwtrKGC7jEM8H29DX3QCM0A707AYWcpp7rHnX73xR0QNuzgExxvtNuBnQEz2tsCAmaOJxI/5h3P5SJzNl9/kzA1j9v+mUg59UJcswtGtCyXl2nNs2oZZNq70jTW0H5kbEbl4PIuudZ8KR4ly7cmpUI+SJFzLp0dvtfe7KJPJ21EOPoyULws6z+G6ju66WHO3wnprsHJGJX1mvBOG8/c83f0J2mp3PnaOupPDvRGIpf9P8oFNpxurrQKrG3sMdbYCGF/Go2FnR77YV0U8WtjtOodMH5JM6AfdEPGumqbpPjrXLWoc0jjX0fmx6lO7ptuVTzTLjMC/ds/9KsTxyP3l1J0qFkxAWFzdpptnqlfei7SdmSfA2xxOzgu/Q7Juve3Yu9DBu9kHf0zSFoY9cUDIHonplpo0w3LVl86qs5kgVmkszOfGbn/I7qSiwijdB2aj7Po2sv05FgJm+TbhtFRBC9pv1pk=
-    username: AgAhWDU81aYqCplI25qaQM8TmlfrEjrHAM/RZfao40b4dpr93rWYAhhehnfH+UpoYftt/epGPZ1Jw6N61ph1+cEOjCdDYT497va4IjL3EYLrjODMQdRqeem06DMGmoWFj6BaQPdA5FuIOJJnqbcFHuyloYHjnWSQVzLLGlc2DLhAPmsIog625+SXjHCRGMFR6Gwsndfc2P47CBshnXNv/t4JgluyIwW0iKMHVsafAJhf5JWc9gldcV0CBkqRf9fRKh777Dz6+SKJJkd+2bD+N7kXr+CBcr7SPuJa+/Hia2S1tbUukjfZ3gjwNz1vzxNBzJ/sbVJM6w0qbESyMtKUYBPEkOx5F3fmsvZ1nihch/pM/Hz+nlk2Z+6+DqZC5yW7qpNE/kBpMFcUgusqXUEJeEwGNs+fcf+thVXXLqqiF/IhSsWBms4C+hOX2ECdzrd2t/gD9XmcflaqhrbkWt596RkTH1sAC8lR6MncwNJk/AtlydQxZHWJNMr5nA+c3AvdSaVgsUTMmC8hJVeOCQaVC9JhtRpqZqQ+RS2pl7SZo829GxCuKuOFkqc1GVudWVJe2R5Xw+CJzwg0xxyt86whXC5If0g6mCRMfd44lFPiHJKa94VZDZkQtA5eaDG++yYoIVbF370g1DTk+S3Lndtsc2KzDPdKYqaWETKdenpEWxadzxHXyakfu98EbUbf04xn6tbi1BvPyTA=
-  template:
-    metadata:
-      name: argocd-ghcr-helm-credentials
-      namespace: argocd

remodel/k8s/infra/controllers/argocd/kustomization.yaml

@@ -3,8 +3,6 @@ kind: Kustomization
 
 resources:
   - ns.yaml
-#  - ghcr-helm-credentials.yaml
-#  - docker-helm-credentials.yaml
 #  - http-route.yaml
 
 helmCharts:

remodel/tofu/kubernetes/bootstrap/sealed-secrets/variables.tf

@@ -3,4 +3,4 @@ variable "sealed_secrets_cert" {
     cert = string
     key = string
   })
-}
+}

remodel/tofu/kubernetes/config/talos-image-schematic.yaml

@@ -0,0 +1,6 @@
+customization:
+  systemExtensions:
+    officialExtensions:
+      - siderolabs/i915-ucode
+      - siderolabs/intel-ucode
+      - siderolabs/qemu-guest-agent

remodel/tofu/kubernetes/main.tf

@@ -5,7 +5,11 @@ module "talos" {
     proxmox = proxmox
   }
 
-  talos_image    = var.talos_image
+  talos_image = {
+    version = "v1.7.5"
+    update_version = "v1.8.0-alpha.1"
+    schematic = file("${path.module}/config/talos-image-schematic.yaml")
+  }
   cluster_config = var.cluster_config
   cilium = {
     values = file("${path.module}/../../k8s/infra/network/cilium/values.yaml")
@@ -33,8 +37,8 @@ module "sealed_secrets" {
 
   // openssl req -x509 -days 365 -nodes -newkey rsa:4096 -keyout sealed-secrets.key -out sealed-secrets.cert -subj "/CN=sealed-secret/O=sealed-secret"
   sealed_secrets_cert = {
-    cert = file("${path.module}/bootstrap/sealed-secrets/sealed-secrets.cert")
+    cert = file("${path.module}/config/sealed-secrets.cert")
-    key = file("${path.module}/bootstrap/sealed-secrets/sealed-secrets.key")
+    key = file("${path.module}/config/sealed-secrets.key")
   }
 }
 

remodel/tofu/kubernetes/talos/image.tf

@@ -1,13 +1,44 @@
-# Download the Talos image to each distinct Proxmox node
+locals {
-resource "proxmox_virtual_environment_download_file" "talos_nocloud_image" {
+  schematic_id = jsondecode(data.http.schematic_id.response_body)["id"]
+  update_schematic = coalesce(var.talos_image.update_schematic, var.talos_image.schematic)
+  update_version = coalesce(var.talos_image.update_version, var.talos_image.version)
+  update_schematic_id = jsondecode(data.http.updated_schematic_id.response_body)["id"]
+}
+
+data "http" "schematic_id" {
+  url          = "${var.talos_image.factory_url}/schematics"
+  method       = "POST"
+  request_body = var.talos_image.schematic
+}
+
+data "http" "updated_schematic_id" {
+  url          = "${var.talos_image.factory_url}/schematics"
+  method       = "POST"
+  request_body = local.update_schematic
+}
+
+resource "proxmox_virtual_environment_download_file" "talos_image" {
   for_each = toset(distinct([for k, v in var.cluster_config.nodes : v.host_node]))
 
   node_name    = each.key
   content_type = "iso"
-  datastore_id = var.talos_image.datastore
+  datastore_id = var.talos_image.proxmox_datastore
 
-  file_name               = "talos-${var.talos_image.version}-nocloud-amd64.img"
+  file_name               = "talos-${local.schematic_id}-${var.talos_image.version}-${var.talos_image.platform}-${var.talos_image.arch}.img"
-  url                     = "${var.talos_image.base_url}/${var.talos_image.version}/nocloud-amd64.raw.gz"
+  url                     = "${var.talos_image.factory_url}/image/${local.schematic_id}/${var.talos_image.version}/${var.talos_image.platform}-${var.talos_image.arch}.raw.gz"
+  decompression_algorithm = "gz"
+  overwrite               = false
+}
+
+resource "proxmox_virtual_environment_download_file" "updated_talos_image" {
+  for_each = toset(distinct([for k, v in var.cluster_config.nodes : v.host_node if v.update]))
+
+  node_name    = each.key
+  content_type = "iso"
+  datastore_id = var.talos_image.proxmox_datastore
+
+  file_name               = "talos-update-${local.update_schematic_id}-${local.update_version}-${var.talos_image.platform}-${var.talos_image.arch}.img"
+  url                     = "${var.talos_image.factory_url}/image/${local.update_schematic_id}/${var.talos_image.update_version}/${var.talos_image.platform}-${var.talos_image.arch}.raw.gz"
   decompression_algorithm = "gz"
   overwrite               = false
 }

remodel/tofu/kubernetes/talos/talos-config.tf

@@ -17,17 +17,17 @@ data "talos_machine_configuration" "machine_configuration" {
   talos_version    = var.cluster_config.talos_version
   config_patches   = each.value.machine_type == "controlplane" ? [
     templatefile("${path.module}/machine-config/control-plane.yaml.tftpl", {
-      hostname = each.key
+      hostname       = each.key
-      cluster_name = var.cluster_config.proxmox_cluster
+      cluster_name   = var.cluster_config.proxmox_cluster
-      node_name = each.value.host_node
+      node_name      = each.value.host_node
-      cilium_values = var.cilium.values
+      cilium_values  = var.cilium.values
       cilium_install = var.cilium.install
     })
   ] : [
     templatefile("${path.module}/machine-config/worker.yaml.tftpl", {
-      hostname = each.key
+      hostname     = each.key
       cluster_name = var.cluster_config.proxmox_cluster
-      node_name = each.value.host_node
+      node_name    = each.value.host_node
     })
   ]
 }
@@ -38,12 +38,16 @@ resource "talos_machine_configuration_apply" "talos_config_apply" {
   node                        = each.value.ip
   client_configuration        = talos_machine_secrets.machine_secrets.client_configuration
   machine_configuration_input = data.talos_machine_configuration.machine_configuration[each.key].machine_configuration
+  lifecycle {
+    # re-run config apply if vm changes
+    replace_triggered_by = [proxmox_virtual_environment_vm.talos_vm[each.key]]
+  }
 }
 
 resource "talos_machine_bootstrap" "talos_bootstrap" {
   depends_on = [talos_machine_configuration_apply.talos_config_apply]
   client_configuration = talos_machine_secrets.machine_secrets.client_configuration
-  node                 = [for k, v in var.cluster_config.nodes : v.ip if v.machine_type == "controlplane"][0]
+  node                 = [for k, v in var.cluster_config.nodes : v.ip if v.machine_type == "controlplane" && !v.update][0]
 }
 
 data "talos_cluster_health" "health" {
@@ -58,8 +62,11 @@ data "talos_cluster_health" "health" {
 }
 
 data "talos_cluster_kubeconfig" "kubeconfig" {
-  #depends_on = [talos_machine_bootstrap.talos_bootstrap]
+  #  depends_on = [talos_machine_bootstrap.talos_bootstrap]
   depends_on = [talos_machine_bootstrap.talos_bootstrap, data.talos_cluster_health.health]
   client_configuration = talos_machine_secrets.machine_secrets.client_configuration
-  node                 = [for k, v in var.cluster_config.nodes : v.ip if v.machine_type == "controlplane"][0]
+  node                 = [for k, v in var.cluster_config.nodes : v.ip if v.machine_type == "controlplane" && !v.update][0]
+  timeouts = {
+    read = "1m"
+  }
 }

remodel/tofu/kubernetes/talos/variables.tf

@@ -1,8 +1,13 @@
 variable "talos_image" {
   type = object({
-    base_url = string
+    factory_url = optional(string, "https://factory.talos.dev")
-    version  = string
+    schematic = string
-    datastore = string
+    version   = string
+    update_schematic = optional(string)
+    update_version = optional(string)
+    arch = optional(string, "amd64")
+    platform = optional(string, "nocloud")
+    proxmox_datastore = optional(string, "local")
   })
 }
 
@@ -23,6 +28,7 @@ variable "cluster_config" {
       vm_id         = number
       cpu           = number
       ram_dedicated = number
+      update = optional(bool, false)
       igpu = optional(bool, false)
     }))
   })
@@ -30,7 +36,7 @@ variable "cluster_config" {
 
 variable "cilium" {
   type = object({
-    values = string
+    values  = string
     install = string
   })
 }

remodel/tofu/kubernetes/talos/virtual-machines.tf

@@ -37,10 +37,10 @@ resource "proxmox_virtual_environment_vm" "talos_vm" {
     iothread     = true
     cache        = "writethrough"
     discard      = "on"
-    ssd = true
+    ssd          = true
-    file_id      = proxmox_virtual_environment_download_file.talos_nocloud_image[each.value.host_node].id
     file_format  = "raw"
     size         = 20
+    file_id      = each.value.update ? proxmox_virtual_environment_download_file.updated_talos_image[each.value.host_node].id : proxmox_virtual_environment_download_file.talos_image[each.value.host_node].id
   }
 
   boot_order = ["scsi0"]
@@ -50,7 +50,7 @@ resource "proxmox_virtual_environment_vm" "talos_vm" {
   }
 
   initialization {
-    datastore_id      = "local-zfs"
+    datastore_id = "local-zfs"
     ip_config {
       ipv4 {
         address = "${each.value.ip}/24"
@@ -66,7 +66,7 @@ resource "proxmox_virtual_environment_vm" "talos_vm" {
     for_each = each.value.igpu ? [1] : []
     content {
       # Passthrough iGPU
-      device = "hostpci0"
+      device  = "hostpci0"
       mapping = "iGPU"
       pcie    = true
       rombar  = true

remodel/tofu/kubernetes/variables.tf

@@ -10,14 +10,6 @@ variable "proxmox" {
   sensitive = true
 }
 
-variable "talos_image" {
-  type = object({
-    base_url  = string
-    version   = string
-    datastore = string
-  })
-}
-
 variable "cluster_config" {
   description = "Talos node configuration"
   type = object({
@@ -36,6 +28,7 @@ variable "cluster_config" {
         vm_id         = number
         cpu           = number
         ram_dedicated = number
+        update = optional(bool, false)
         igpu = optional(bool, false)
       })
     )