karpenter integration

proxmox/common.tf

@@ -18,6 +18,24 @@ resource "proxmox_virtual_environment_download_file" "talos" {
   url                     = "https://factory.talos.dev/image/14e9b0100f05654bedf19b92313cdc224cbff52879193d24f3741f1da4a3cbb1/v${var.release}/nocloud-amd64.raw.xz"
 }
 
+resource "proxmox_virtual_environment_file" "machineconfig" {
+  for_each     = { for inx, zone in local.zones : zone => inx if lookup(try(var.instances[zone], {}), "enabled", false) }
+  node_name    = each.key
+  content_type = "snippets"
+  datastore_id = "local"
+
+  source_raw {
+    data = templatefile("${path.module}/templates/common.yaml.tpl",
+      merge(local.kubernetes, try(var.instances["all"], {}), {
+        labels      = "node-pool=common,karpenter.sh/nodepool=default"
+        nodeSubnets = [var.vpc_main_cidr[0], var.vpc_main_cidr[1]]
+        lbv4        = local.lbv4
+        kernelArgs  = []
+    }))
+    file_name = "common.yaml"
+  }
+}
+
 resource "proxmox_virtual_environment_vm" "template" {
   for_each    = { for inx, zone in local.zones : zone => inx if lookup(try(var.instances[zone], {}), "enabled", false) }
   name        = "talos"
@@ -50,10 +68,40 @@ resource "proxmox_virtual_environment_vm" "template" {
     file_format  = "raw"
   }
 
+  network_device {
+    bridge   = "vmbr0"
+    mtu      = 1500
+    firewall = true
+  }
+  network_device {
+    bridge   = "vmbr1"
+    mtu      = 1400
+    firewall = false
+  }
+
   operating_system {
     type = "l26"
   }
 
+  initialization {
+    dns {
+      servers = ["1.1.1.1", "2001:4860:4860::8888"]
+    }
+    ip_config {
+      ipv6 {
+        address = "auto"
+      }
+    }
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+    }
+
+    datastore_id      = "local"
+    user_data_file_id = proxmox_virtual_environment_file.machineconfig[each.key].id
+  }
+
   serial_device {}
   vga {
     type = "serial0"

proxmox/deployments/talos-ccm.yaml

@@ -1,4 +1,3 @@
-
 image:
   # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager
   pullPolicy: Always
@@ -59,3 +58,14 @@ transformations:
               - ^db-.+$
     labels:
       node-role.kubernetes.io/db: ""
+
+  - name: nocloud
+    nodeSelector:
+      - matchExpressions:
+          - key: platform
+            operator: In
+            values:
+              - nocloud
+    platformMetadata:
+      InstanceType: "{{ coalesce .InstanceType .SKUNumber }}"
+      ProviderID: '{{ if .SerialNumber }}proxmox://region-1/{{ getValue .SerialNumber "i" }}{{ else }}{{ .ProviderID }}{{ end }}'

proxmox/templates/common.yaml.tpl

@@ -0,0 +1,91 @@
+version: v1alpha1
+debug: false
+persist: true
+machine:
+  type: worker
+  token: ${tokenMachine}
+  ca:
+    crt: ${caMachine}
+  kubelet:
+    image: ghcr.io/siderolabs/kubelet:${version}
+    defaultRuntimeSeccompProfileEnabled: true
+    extraArgs:
+      cloud-provider: external
+      rotate-server-certificates: true
+      node-labels: ${labels}
+      register-with-taints: "karpenter.sh/unregistered=:NoExecute"
+    extraConfig:
+      imageGCHighThresholdPercent: 70
+      imageGCLowThresholdPercent: 50
+      shutdownGracePeriod: 60s
+      topologyManagerPolicy: best-effort
+      topologyManagerScope: container
+      cpuManagerPolicy: static
+      allowedUnsafeSysctls: [net.core.somaxconn]
+    clusterDNS:
+      - 169.254.2.53
+      - ${cidrhost(split(",",serviceSubnets)[0], 10)}
+    nodeIP:
+      validSubnets: ${format("%#v",nodeSubnets)}
+  network:
+    interfaces:
+      - interface: dummy0
+        addresses:
+          - 169.254.2.53/32
+    extraHostEntries:
+      - ip: ${lbv4}
+        aliases:
+          - ${apiDomain}
+  sysctls:
+    net.core.somaxconn: 65535
+    net.core.netdev_max_backlog: 4096
+    net.ipv4.tcp_keepalive_intvl: 60
+    net.ipv4.tcp_keepalive_time: 600
+    net.ipv4.tcp_fin_timeout: 10
+    net.ipv4.tcp_tw_reuse: 1
+    vm.max_map_count: 128000
+  install:
+    wipe: true
+    extraKernelArgs:
+      - talos.dashboard.disabled=1
+%{ for arg in kernelArgs ~}
+      - ${arg}
+%{ endfor ~}
+  systemDiskEncryption:
+    state:
+      provider: luks2
+      options:
+        - no_read_workqueue
+        - no_write_workqueue
+      keys:
+        - nodeID: {}
+          slot: 0
+    ephemeral:
+      provider: luks2
+      options:
+        - no_read_workqueue
+        - no_write_workqueue
+      keys:
+        - nodeID: {}
+          slot: 0
+  features:
+    rbac: true
+    stableHostname: true
+    apidCheckExtKeyUsage: true
+cluster:
+  id: ${clusterID}
+  secret: ${clusterSecret}
+  controlPlane:
+    endpoint: https://${apiDomain}:6443
+  clusterName: ${clusterName}
+  discovery:
+    enabled: false
+  network:
+    dnsDomain: ${domain}
+    podSubnets: ${format("%#v",split(",",podSubnets))}
+    serviceSubnets: ${format("%#v",split(",",serviceSubnets))}
+  proxy:
+    disabled: true
+  token: ${token}
+  ca:
+    crt: ${ca}

proxmox/variables.tf

@@ -26,7 +26,7 @@ variable "vpc_main_cidr" {
 variable "release" {
   type        = string
   description = "The version of the Talos image"
-  default     = "1.8.3"
+  default     = "1.8.4"
 }
 
 data "sops_file" "tfvars" {
@@ -86,7 +86,7 @@ variable "instances" {
   type        = map(any)
   default = {
     "all" = {
-      version = "v1.31.3"
+      version = "v1.31.4"
     },
     "hvm-1" = {
       enabled         = false,