From e343d41b8547ce35a263b60f19dc66ad854ba526 Mon Sep 17 00:00:00 2001 From: Vegard Hagen Date: Sat, 16 Mar 2024 20:35:56 +0100 Subject: [PATCH] feat(cloud-init): Kubeadm cluster wip: DNS shenanigans --- .gitignore | 1 + PROXMOX.md | 10 +- machines/euclid/cloud-init/control-plane.yaml | 86 ++++++++++++++ .../cloud-init/{user.yaml => worker.yaml} | 30 ++++- machines/euclid/images/download.sh | 4 +- machines/euclid/variables.tf | 10 ++ machines/euclid/vm-home-assistant.tf | 4 +- machines/euclid/vm-k8s.tf | 110 ++++++++++++------ 8 files changed, 206 insertions(+), 49 deletions(-) create mode 100644 machines/euclid/cloud-init/control-plane.yaml rename machines/euclid/cloud-init/{user.yaml => worker.yaml} (61%) diff --git a/.gitignore b/.gitignore index 241b384..a1950ca 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ charts/example *secret*.yaml **/.terraform/* +**/output *.tfstate *.tfstate.* diff --git a/PROXMOX.md b/PROXMOX.md index 0d20c2e..28d01cb 100644 --- a/PROXMOX.md +++ b/PROXMOX.md @@ -35,8 +35,6 @@ dmesg | grep -e DMAR -e IOMMU DMAR: IOMMU enabled ``` - - Nvidia ```shell echo "blacklist nouveau" >> /etc/modprobe.d/blacklist.conf @@ -53,10 +51,16 @@ pvesh get /nodes//hardware/pci --pci-class-blacklist "" https://3os.org/infrastructure/proxmox/gpu-passthrough/igpu-passthrough-to-vm/#linux-virtual-machine-igpu-passthrough-configuration +In Guest VM + ```shell - sudo lspci -nnv | grep VGA +sudo lspci -nnv | grep VGA ``` +Mapped device +https://pve.proxmox.com/pve-docs/pve-admin-guide.html#resource_mapping + + ## Pass through Disk https://pve.proxmox.com/wiki/Passthrough_Physical_Disk_to_Virtual_Machine_(VM) diff --git a/machines/euclid/cloud-init/control-plane.yaml b/machines/euclid/cloud-init/control-plane.yaml new file mode 100644 index 0000000..098ca5d --- /dev/null +++ b/machines/euclid/cloud-init/control-plane.yaml @@ -0,0 +1,86 @@ +#cloud-config +users: + - name: ${username} + groups: + - sudo + shell: /bin/bash + ssh_authorized_keys: + - ${pub-key} + sudo: ALL=(ALL) NOPASSWD:ALL + +network: + version: 1 + config: + - type: nameserver + address: + - 1.1.1.1 + - 8.8.8.8 + +hostname: ${hostname} +create_hostname_file: true +package_update: true +package_upgrade: true +locale: en_US.UTF-8 +timezone: Europe/Oslo + +write_files: + - path: /etc/modules-load.d/k8s.conf + content: | + overlay + br_netfilter + + - path: /etc/sysctl.d/k8s.conf + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + net.ipv4.ip_forward = 1 + # https://serverfault.com/questions/1148659/overwriting-provider-dns-via-cloud-init + - path: /etc/systemd/resolved.conf.d/dns_servers.conf + content: | + [Resolve] + DNS=1.1.1.1 8.8.8.8 + Domains=~. + permissions: '0644' + +packages: + - qemu-guest-agent + - net-tools + - vim + - apt-transport-https + - ca-certificates + - curl + - gpg + - open-iscsi + - jq + +runcmd: + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - localectl set-locale LANG=en_US.UTF-8 + - curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + - apt update + - apt install -y kubelet kubeadm kubectl + - apt-mark hold kubelet kubeadm kubectl + - apt install -y runc containerd + - containerd config default | tee /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + - modprobe overlay + - modprobe br_netfilter + - sysctl --system + - systemctl restart containerd + - systemctl restart systemd-resolved + - ${kubeadm-cmd} + - mkdir -p /home/${username}/.kube + - cp /etc/kubernetes/admin.conf /home/${username}/.kube/config + - chown -R ${username}:${username} /home/${username}/.kube + - curl -sfLO --fail https://github.com/cilium/cilium-cli/releases/download/v${cilium-cli-version}/cilium-linux-amd64.tar.gz + - tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin + - rm cilium-linux-amd64.tar.gz + - ${cilium-cli-cmd} + +power_state: + delay: now + mode: reboot + message: Rebooting after cloud-init completion + condition: true \ No newline at end of file diff --git a/machines/euclid/cloud-init/user.yaml b/machines/euclid/cloud-init/worker.yaml similarity index 61% rename from machines/euclid/cloud-init/user.yaml rename to machines/euclid/cloud-init/worker.yaml index 574aff1..afb42cd 100644 --- a/machines/euclid/cloud-init/user.yaml +++ b/machines/euclid/cloud-init/worker.yaml @@ -5,10 +5,17 @@ users: - sudo shell: /bin/bash ssh_authorized_keys: - - ${pub_key} + - ${pub-key} sudo: ALL=(ALL) NOPASSWD:ALL -manage_etc_hosts: true +network: + version: 1 + config: + - type: nameserver + address: + - 1.1.1.1 + - 8.8.8.8 + hostname: ${hostname} create_hostname_file: true package_update: true @@ -27,6 +34,13 @@ write_files: net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 net.ipv4.ip_forward = 1 + # https://serverfault.com/questions/1148659/overwriting-provider-dns-via-cloud-init + - path: /etc/systemd/resolved.conf.d/dns_servers.conf + content: | + [Resolve] + DNS=1.1.1.1 8.8.8.8 + Domains=~. + permissions: '0644' packages: - qemu-guest-agent @@ -43,8 +57,8 @@ runcmd: - systemctl enable qemu-guest-agent - systemctl start qemu-guest-agent - localectl set-locale LANG=en_US.UTF-8 - - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.29/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg - - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.29/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list + - curl -fsSL https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + - echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${k8s-version}/deb/ /' | tee /etc/apt/sources.list.d/kubernetes.list - apt update - apt install -y kubelet kubeadm kubectl - apt-mark hold kubelet kubeadm kubectl @@ -55,3 +69,11 @@ runcmd: - modprobe br_netfilter - sysctl --system - systemctl restart containerd + - systemctl restart systemd-resolved + - ${kubeadm-cmd} + +power_state: + delay: now + mode: reboot + message: Rebooting after cloud-init completion + condition: true \ No newline at end of file diff --git a/machines/euclid/images/download.sh b/machines/euclid/images/download.sh index c3d978e..033372f 100755 --- a/machines/euclid/images/download.sh +++ b/machines/euclid/images/download.sh @@ -1,2 +1,2 @@ -wget https://github.com/home-assistant/operating-system/releases/download/12.0/haos_ova-12.0.qcow2.xz -xz -d haos_ova-12.0.qcow2.xz \ No newline at end of file +wget https://github.com/home-assistant/operating-system/releases/download/12.1/haos_ova-12.1.qcow2.xz +xz -d haos_ova-12.1.qcow2.xz \ No newline at end of file diff --git a/machines/euclid/variables.tf b/machines/euclid/variables.tf index 6ab7ba5..bcbcad0 100644 --- a/machines/euclid/variables.tf +++ b/machines/euclid/variables.tf @@ -26,3 +26,13 @@ variable "vm_pub-key" { description = "vm username" type = string } + +variable "k8s-version" { + description = "Kubernetes version" + type = string +} + +variable "cilium-cli-version" { + description = "Cilium CLI version" + type = string +} diff --git a/machines/euclid/vm-home-assistant.tf b/machines/euclid/vm-home-assistant.tf index e5c126e..f8d20ba 100644 --- a/machines/euclid/vm-home-assistant.tf +++ b/machines/euclid/vm-home-assistant.tf @@ -5,8 +5,8 @@ resource "proxmox_virtual_environment_file" "haos_generic_image" { datastore_id = "local" source_file { - path = "images/haos_ova-12.0.qcow2" - file_name = "haos_ova-12.0.img" + path = "images/haos_ova-12.1.qcow2" + file_name = "haos_ova-12.1.img" } } diff --git a/machines/euclid/vm-k8s.tf b/machines/euclid/vm-k8s.tf index 23d1a1e..2f3f72e 100644 --- a/machines/euclid/vm-k8s.tf +++ b/machines/euclid/vm-k8s.tf @@ -19,31 +19,19 @@ resource "proxmox_virtual_environment_file" "cloud-init-ctrl-01" { datastore_id = "local" source_raw { - data = templatefile("./cloud-init/user.yaml", { - username = var.vm_user - pub_key = var.vm_pub-key - hostname = "k8s-ctrl-01" + data = templatefile("./cloud-init/control-plane.yaml", { + hostname = "k8s-ctrl-01" + username = var.vm_user + pub-key = var.vm_pub-key + k8s-version = var.k8s-version + kubeadm-cmd = "kubeadm init --skip-phases=addon/kube-proxy" + cilium-cli-version = var.cilium-cli-version + cilium-cli-cmd = "KUBECONFIG=/etc/kubernetes/admin.conf cilium install --set kubeProxyReplacement=true" }) file_name = "cloud-init-k8s-ctrl-01.yaml" } } -resource "proxmox_virtual_environment_file" "cloud-init-work-01" { - provider = proxmox.euclid - node_name = var.euclid.node_name - content_type = "snippets" - datastore_id = "local" - - source_raw { - data = templatefile("./cloud-init/user.yaml", { - username = var.vm_user - pub_key = var.vm_pub-key - hostname = "k8s-work-01" - }) - file_name = "cloud-init-k8s-work-01.yaml" - } -} - resource "proxmox_virtual_environment_vm" "k8s-ctrl-01" { provider = proxmox.euclid node_name = var.euclid.node_name @@ -113,6 +101,63 @@ resource "proxmox_virtual_environment_vm" "k8s-ctrl-01" { } } +output "ctrl_01_ipv4_address" { + depends_on = [proxmox_virtual_environment_vm.k8s-ctrl-01] + value = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0] +} + +resource "local_file" "ctrl-01-ip" { + content = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0] + filename = "output/ctrl-01-ip.txt" + file_permission = "0644" +} + +module "sleep" { + depends_on = [local_file.ctrl-01-ip] + source = "Invicton-Labs/shell-data/external" + version = "0.4.2" + command_unix = "sleep 120" +} + +module "kube-config" { + depends_on = [module.sleep] + source = "Invicton-Labs/shell-resource/external" + version = "0.4.1" + command_unix = "ssh -o StrictHostKeyChecking=no ${var.vm_user}@${local_file.ctrl-01-ip.content} cat /home/${var.vm_user}/.kube/config" +} + +resource "local_file" "kube-config" { + content = module.kube-config.stdout + filename = "output/config" + file_permission = "0600" +} + +module "kubeadm-join" { + depends_on = [local_file.kube-config] + source = "Invicton-Labs/shell-resource/external" + version = "0.4.1" + # https://stackoverflow.com/questions/21383806/how-can-i-force-ssh-to-accept-a-new-host-fingerprint-from-the-command-line + command_unix = "ssh -o StrictHostKeyChecking=no ${var.vm_user}@${local_file.ctrl-01-ip.content} /usr/bin/kubeadm token create --print-join-command" +} + +resource "proxmox_virtual_environment_file" "cloud-init-work-01" { + provider = proxmox.euclid + node_name = var.euclid.node_name + content_type = "snippets" + datastore_id = "local" + + source_raw { + data = templatefile("./cloud-init/worker.yaml", { + hostname = "k8s-work-01" + username = var.vm_user + pub-key = var.vm_pub-key + k8s-version = var.k8s-version + kubeadm-cmd = module.kubeadm-join.stdout + }) + file_name = "cloud-init-k8s-work-01.yaml" + } +} + resource "proxmox_virtual_environment_vm" "k8s-work-01" { provider = proxmox.euclid node_name = var.euclid.node_name @@ -182,18 +227,13 @@ resource "proxmox_virtual_environment_vm" "k8s-work-01" { hostpci { # Passthrough iGPU - device = "hostpci0" - id = "0000:00:02" - pcie = true - rombar = true - xvga = false + device = "hostpci0" + #id = "0000:00:02" + mapping = "iGPU" + pcie = true + rombar = true + xvga = false } - -} - -output "ctrl_01_ipv4_address" { - depends_on = [proxmox_virtual_environment_vm.k8s-ctrl-01] - value = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0] } output "work_01_ipv4_address" { @@ -201,13 +241,7 @@ output "work_01_ipv4_address" { value = proxmox_virtual_environment_vm.k8s-work-01.ipv4_addresses[1][0] } -resource "local_file" "ctrl_01_ip" { - content = proxmox_virtual_environment_vm.k8s-ctrl-01.ipv4_addresses[1][0] - filename = "output/ctrl-01-ip.txt" - file_permission = "0644" -} - -resource "local_file" "work_01_ip" { +resource "local_file" "work-01-ip" { content = proxmox_virtual_environment_vm.k8s-work-01.ipv4_addresses[1][0] filename = "output/work-01-ip.txt" file_permission = "0644"