feat(cloud-init): Kubeadm cluster

wip: DNS shenanigans

.gitignore

@@ -4,6 +4,7 @@ charts/example
 *secret*.yaml
 
 **/.terraform/*
+**/output
 
 *.tfstate
 *.tfstate.*

PROXMOX.md

@@ -35,8 +35,6 @@ dmesg | grep -e DMAR -e IOMMU
 DMAR: IOMMU enabled
 ```
 
-
-
 Nvidia
 ```shell
 echo "blacklist nouveau" >> /etc/modprobe.d/blacklist.conf 
@@ -53,10 +51,16 @@ pvesh get /nodes/<NODE_NAME>/hardware/pci --pci-class-blacklist ""
 
 https://3os.org/infrastructure/proxmox/gpu-passthrough/igpu-passthrough-to-vm/#linux-virtual-machine-igpu-passthrough-configuration
 
+In Guest VM
+
 ```shell
- sudo lspci -nnv | grep VGA
+sudo lspci -nnv | grep VGA
 ```
 
+Mapped device
+https://pve.proxmox.com/pve-docs/pve-admin-guide.html#resource_mapping
+
+
 ## Pass through Disk
 https://pve.proxmox.com/wiki/Passthrough_Physical_Disk_to_Virtual_Machine_(VM)
 

machines/euclid/cloud-init/control-plane.yaml

@@ -0,0 +1,86 @@
+#cloud-config
+users:
+  - name: ${username}
+    groups:
+      - sudo
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - ${pub-key}
+    sudo: ALL=(ALL) NOPASSWD:ALL
+
+network:
+  version: 1
+  config:
+    - type: nameserver
+      address:
+        - 1.1.1.1
+        - 8.8.8.8
+
+hostname: ${hostname}
+create_hostname_file: true
+package_update: true
+package_upgrade: true
+locale: en_US.UTF-8
+timezone: Europe/Oslo
+
+write_files:
+  - path: /etc/modules-load.d/k8s.conf
+    content: |
+      overlay
+      br_netfilter
+
+  - path: /etc/sysctl.d/k8s.conf
+    content: |
+      net.bridge.bridge-nf-call-ip6tables = 1
+      net.bridge.bridge-nf-call-iptables  = 1
+      net.ipv4.ip_forward                 = 1
+  # https://serverfault.com/questions/1148659/overwriting-provider-dns-via-cloud-init
+  - path: /etc/systemd/resolved.conf.d/dns_servers.conf
+    content: |
+      [Resolve]
+      DNS=1.1.1.1 8.8.8.8
+      Domains=~.
+    permissions: '0644'
+
+packages:
+  - qemu-guest-agent
+  - net-tools
+  - vim
+  - apt-transport-https
+  - ca-certificates
+  - curl
+  - gpg
+  - open-iscsi
+  - jq
+
+runcmd:
+  - systemctl enable qemu-guest-agent
+  - systemctl start qemu-guest-agent
+  - localectl set-locale LANG=en_US.UTF-8
+  - curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+  - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list
+  - apt update
+  - apt install -y kubelet kubeadm kubectl
+  - apt-mark hold kubelet kubeadm kubectl
+  - apt install -y runc containerd
+  - containerd config default | tee /etc/containerd/config.toml
+  - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml
+  - modprobe overlay
+  - modprobe br_netfilter
+  - sysctl --system
+  - systemctl restart containerd
+  - systemctl restart systemd-resolved
+  - ${kubeadm-cmd}
+  - mkdir -p /home/${username}/.kube
+  - cp /etc/kubernetes/admin.conf /home/${username}/.kube/config
+  - chown -R ${username}:${username} /home/${username}/.kube
+  - curl -sfLO --fail https://github.com/cilium/cilium-cli/releases/download/v${cilium-cli-version}/cilium-linux-amd64.tar.gz
+  - tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin
+  - rm cilium-linux-amd64.tar.gz
+  - ${cilium-cli-cmd}
+
+power_state:
+  delay: now
+  mode: reboot
+  message: Rebooting after cloud-init completion
+  condition: true

machines/euclid/cloud-init/worker.yaml

@@ -5,10 +5,17 @@ users:
       - sudo
     shell: /bin/bash
     ssh_authorized_keys:
-      - ${pub_key}
+      - ${pub-key}
     sudo: ALL=(ALL) NOPASSWD:ALL
 
-manage_etc_hosts: true
+network:
+  version: 1
+  config:
+    - type: nameserver
+      address:
+        - 1.1.1.1
+        - 8.8.8.8
+
 hostname: ${hostname}
 create_hostname_file: true
 package_update: true
@@ -27,6 +34,13 @@ write_files:
       net.bridge.bridge-nf-call-ip6tables = 1
       net.bridge.bridge-nf-call-iptables  = 1
       net.ipv4.ip_forward                 = 1
+  # https://serverfault.com/questions/1148659/overwriting-provider-dns-via-cloud-init
+  - path: /etc/systemd/resolved.conf.d/dns_servers.conf
+    content: |
+      [Resolve]
+      DNS=1.1.1.1 8.8.8.8
+      Domains=~.
+    permissions: '0644'
 
 packages:
   - qemu-guest-agent
@@ -43,8 +57,8 @@ runcmd:
   - systemctl enable qemu-guest-agent
   - systemctl start qemu-guest-agent
   - localectl set-locale LANG=en_US.UTF-8
-  - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+  - curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
-  - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list
+  - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list
   - apt update
   - apt install -y kubelet kubeadm kubectl
   - apt-mark hold kubelet kubeadm kubectl
@@ -55,3 +69,11 @@ runcmd:
   - modprobe br_netfilter
   - sysctl --system
   - systemctl restart containerd
+  - systemctl restart systemd-resolved
+  - ${kubeadm-cmd}
+
+power_state:
+  delay: now
+  mode: reboot
+  message: Rebooting after cloud-init completion
+  condition: true

machines/euclid/images/download.sh

@@ -1,2 +1,2 @@
-wget https://github.com/home-assistant/operating-system/releases/download/12.0/haos_ova-12.0.qcow2.xz
+wget https://github.com/home-assistant/operating-system/releases/download/12.1/haos_ova-12.1.qcow2.xz
-xz -d haos_ova-12.0.qcow2.xz
+xz -d haos_ova-12.1.qcow2.xz

machines/euclid/variables.tf

@@ -26,3 +26,13 @@ variable "vm_pub-key" {
   description = "vm username"
   type        = string
 }
+
+variable "k8s-version" {
+  description = "Kubernetes version"
+  type = string
+}
+
+variable "cilium-cli-version" {
+  description = "Cilium CLI version"
+  type = string
+}

machines/euclid/vm-home-assistant.tf

@@ -5,8 +5,8 @@ resource "proxmox_virtual_environment_file" "haos_generic_image" {
   datastore_id = "local"
 
   source_file {
-    path      = "images/haos_ova-12.0.qcow2"
+    path      = "images/haos_ova-12.1.qcow2"
-    file_name = "haos_ova-12.0.img"
+    file_name = "haos_ova-12.1.img"
   }
 }
 

machines/euclid/vm-k8s.tf

@@ -19,31 +19,19 @@ resource "proxmox_virtual_environment_file" "cloud-init-ctrl-01" {
   datastore_id = "local"
 
   source_raw {
-    data = templatefile("./cloud-init/user.yaml", {
+    data = templatefile("./cloud-init/control-plane.yaml", {
-      username = var.vm_user
-      pub_key  = var.vm_pub-key
       hostname           = "k8s-ctrl-01"
+      username           = var.vm_user
+      pub-key            = var.vm_pub-key
+      k8s-version        = var.k8s-version
+      kubeadm-cmd        = "kubeadm init --skip-phases=addon/kube-proxy"
+      cilium-cli-version = var.cilium-cli-version
+      cilium-cli-cmd     = "KUBECONFIG=/etc/kubernetes/admin.conf cilium install --set kubeProxyReplacement=true"
     })
     file_name = "cloud-init-k8s-ctrl-01.yaml"
   }
 }
 
-resource "proxmox_virtual_environment_file" "cloud-init-work-01" {
-  provider     = proxmox.euclid
-  node_name    = var.euclid.node_name
-  content_type = "snippets"
-  datastore_id = "local"
-
-  source_raw {
-    data = templatefile("./cloud-init/user.yaml", {
-      username = var.vm_user
-      pub_key  = var.vm_pub-key
-      hostname = "k8s-work-01"
-    })
-    file_name = "cloud-init-k8s-work-01.yaml"
-  }
-}
-
 resource "proxmox_virtual_environment_vm" "k8s-ctrl-01" {
   provider  = proxmox.euclid
   node_name = var.euclid.node_name
@@ -113,6 +101,63 @@ resource "proxmox_virtual_environment_vm" "k8s-ctrl-01" {
   }
 }
 
+output "ctrl_01_ipv4_address" {
+  depends_on = [proxmox_virtual_environment_vm.k8s-ctrl-01]
+  value      = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0]
+}
+
+resource "local_file" "ctrl-01-ip" {
+  content         = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0]
+  filename        = "output/ctrl-01-ip.txt"
+  file_permission = "0644"
+}
+
+module "sleep" {
+  depends_on   = [local_file.ctrl-01-ip]
+  source       = "Invicton-Labs/shell-data/external"
+  version      = "0.4.2"
+  command_unix = "sleep 120"
+}
+
+module "kube-config" {
+  depends_on   = [module.sleep]
+  source       = "Invicton-Labs/shell-resource/external"
+  version      = "0.4.1"
+  command_unix = "ssh -o StrictHostKeyChecking=no ${var.vm_user}@${local_file.ctrl-01-ip.content} cat /home/${var.vm_user}/.kube/config"
+}
+
+resource "local_file" "kube-config" {
+  content         = module.kube-config.stdout
+  filename        = "output/config"
+  file_permission = "0600"
+}
+
+module "kubeadm-join" {
+  depends_on   = [local_file.kube-config]
+  source       = "Invicton-Labs/shell-resource/external"
+  version      = "0.4.1"
+  # https://stackoverflow.com/questions/21383806/how-can-i-force-ssh-to-accept-a-new-host-fingerprint-from-the-command-line
+  command_unix = "ssh -o StrictHostKeyChecking=no ${var.vm_user}@${local_file.ctrl-01-ip.content} /usr/bin/kubeadm token create --print-join-command"
+}
+
+resource "proxmox_virtual_environment_file" "cloud-init-work-01" {
+  provider     = proxmox.euclid
+  node_name    = var.euclid.node_name
+  content_type = "snippets"
+  datastore_id = "local"
+
+  source_raw {
+    data = templatefile("./cloud-init/worker.yaml", {
+      hostname    = "k8s-work-01"
+      username    = var.vm_user
+      pub-key     = var.vm_pub-key
+      k8s-version = var.k8s-version
+      kubeadm-cmd = module.kubeadm-join.stdout
+    })
+    file_name = "cloud-init-k8s-work-01.yaml"
+  }
+}
+
 resource "proxmox_virtual_environment_vm" "k8s-work-01" {
   provider  = proxmox.euclid
   node_name = var.euclid.node_name
@@ -183,17 +228,12 @@ resource "proxmox_virtual_environment_vm" "k8s-work-01" {
   hostpci {
     # Passthrough iGPU
     device  = "hostpci0"
-    id     = "0000:00:02"
+    #id     = "0000:00:02"
+    mapping = "iGPU"
     pcie    = true
     rombar  = true
     xvga    = false
   }
-
-}
-
-output "ctrl_01_ipv4_address" {
-  depends_on = [proxmox_virtual_environment_vm.k8s-ctrl-01]
-  value      = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0]
 }
 
 output "work_01_ipv4_address" {
@@ -201,13 +241,7 @@ output "work_01_ipv4_address" {
   value      = proxmox_virtual_environment_vm.k8s-work-01.ipv4_addresses[1][0]
 }
 
-resource "local_file" "ctrl_01_ip" {
+resource "local_file" "work-01-ip" {
-  content         = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0]
-  filename        = "output/ctrl-01-ip.txt"
-  file_permission = "0644"
-}
-
-resource "local_file" "work_01_ip" {
   content         = proxmox_virtual_environment_vm.k8s-work-01.ipv4_addresses[1][0]
   filename        = "output/work-01-ip.txt"
   file_permission = "0644"