From 582e8cc035317ba0e403484229c61fe8f42c8861 Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Tue, 4 Apr 2023 16:26:00 +0300 Subject: [PATCH] bootstrap full cluster --- proxmox/Makefile | 3 +- proxmox/README.md | 4 ++ proxmox/common.tf | 6 ++- proxmox/instances-controlplane.tf | 17 ++++---- proxmox/instances-web.tf | 14 ++++-- proxmox/instances-worker.tf | 21 +++++++-- proxmox/outputs.tf | 8 ++++ proxmox/templates/controlplane.yaml.tpl | 2 + proxmox/templates/web.yaml.tpl | 58 +++++++++++++++++++++++++ proxmox/templates/worker.patch.yaml.tpl | 45 +++++++++++++++++++ proxmox/templates/worker.yaml.tpl | 34 +++++++++++---- proxmox/variables.tf | 4 ++ system_os/proxmox/proxmox.pkr.hcl | 6 +-- 13 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 proxmox/templates/web.yaml.tpl create mode 100644 proxmox/templates/worker.patch.yaml.tpl diff --git a/proxmox/Makefile b/proxmox/Makefile index 018fe32..ee960f5 100644 --- a/proxmox/Makefile +++ b/proxmox/Makefile @@ -6,6 +6,7 @@ help: @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) create-config: ## Genereate talos configs + terraform apply -auto-approve -target=local_file.worker_patch talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false --config-patch-worker @templates/worker.patch.yaml ${CLUSTERNAME} https://${ENDPOINT}:6443 talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT} @@ -30,7 +31,7 @@ create-controlplane-bootstrap: talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 bootstrap create-controlplane: ## Bootstrap first controlplane node - terraform apply -auto-approve -target=hcloud_server.controlplane + terraform apply -auto-approve -target=local_file.worker_patch create-kubeconfig: ## Prepare kubeconfig talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.48 kubeconfig . diff --git a/proxmox/README.md b/proxmox/README.md index 2ea9e34..71a4f00 100644 --- a/proxmox/README.md +++ b/proxmox/README.md @@ -17,3 +17,7 @@ cicustom: user=local:snippets/VMID.yaml,meta=local:snippets/VMID.meta ipconfig0: ... net0: ... ``` + +```shell +make create-config create-templates +``` diff --git a/proxmox/common.tf b/proxmox/common.tf index ffb5e75..3145f7d 100644 --- a/proxmox/common.tf +++ b/proxmox/common.tf @@ -1,11 +1,13 @@ resource "local_file" "worker_patch" { - content = templatefile("${path.module}/templates/worker.yaml.tpl", + content = templatefile("${path.module}/templates/worker.patch.yaml.tpl", merge(var.kubernetes, { lbv4 = local.ipv4_vip nodeSubnets = var.vpc_main_cidr + labels = "project.io/node-pool=worker" }) ) - filename = "${path.module}/templates/worker.patch.yaml" + + filename = "${path.module}/templates/worker.patch.yaml.tpl" file_permission = "0600" } diff --git a/proxmox/instances-controlplane.tf b/proxmox/instances-controlplane.tf index eb4280e..b0d0868 100644 --- a/proxmox/instances-controlplane.tf +++ b/proxmox/instances-controlplane.tf @@ -10,6 +10,7 @@ locals { node_name : zone cpu : lookup(try(var.controlplane[zone], {}), "cpu", 1) mem : lookup(try(var.controlplane[zone], {}), "mem", 2048) + ip0 : lookup(try(var.controlplane[zone], {}), "ip0", "ip6=auto") ipv4 : "${cidrhost(local.controlplane_subnet, index(local.zones, zone) + inx)}/24" gwv4 : local.gwv4 } @@ -52,7 +53,7 @@ resource "proxmox_vm_qemu" "controlplane" { define_connection_info = false os_type = "ubuntu" qemu_os = "l26" - ipconfig0 = "ip6=auto" + ipconfig0 = each.value.ip0 ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage @@ -122,10 +123,10 @@ resource "local_file" "controlplane" { file_permission = "0600" } -resource "null_resource" "controlplane" { - for_each = local.controlplanes - provisioner "local-exec" { - command = "echo talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml" - } - depends_on = [proxmox_vm_qemu.controlplane, local_file.controlplane] -} +# resource "null_resource" "controlplane" { +# for_each = local.controlplanes +# provisioner "local-exec" { +# command = "sleep 60 && talosctl apply-config --insecure --nodes ${split("/", each.value.ipv4)[0]} --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml" +# } +# depends_on = [proxmox_vm_qemu.controlplane, local_file.controlplane] +# } diff --git a/proxmox/instances-web.tf b/proxmox/instances-web.tf index 608b9b8..d5f1edc 100644 --- a/proxmox/instances-web.tf +++ b/proxmox/instances-web.tf @@ -12,6 +12,7 @@ locals { node_name : zone cpu : lookup(try(var.instances[zone], {}), "web_cpu", 1) mem : lookup(try(var.instances[zone], {}), "web_mem", 2048) + ip0 : lookup(try(var.instances[zone], {}), "web_ip0", "ip6=auto") ipv4 : "${cidrhost(local.subnets[zone], inx)}/24" gwv4 : local.gwv4 } @@ -28,12 +29,19 @@ resource "null_resource" "web_machineconfig" { } provisioner "file" { - source = "${path.module}/_cfgs/worker.yaml" + # source = "${path.module}/_cfgs/worker.yaml" + content = templatefile("${path.module}/templates/web.yaml.tpl", + merge(var.kubernetes, { + lbv4 = local.ipv4_vip + nodeSubnets = var.vpc_main_cidr + labels = local.web_labels + })) + destination = "/var/lib/vz/snippets/${local.web_prefix}.yaml" } triggers = { - params = filemd5("${path.module}/_cfgs/worker.yaml") + params = filemd5("${path.module}/templates/web.yaml.tpl") } } @@ -72,7 +80,7 @@ resource "proxmox_vm_qemu" "web" { define_connection_info = false os_type = "ubuntu" qemu_os = "l26" - ipconfig0 = "ip6=auto" + ipconfig0 = each.value.ip0 ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "user=local:snippets/${local.web_prefix}.yaml,meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage diff --git a/proxmox/instances-worker.tf b/proxmox/instances-worker.tf index f3c19e2..3f4d0a9 100644 --- a/proxmox/instances-worker.tf +++ b/proxmox/instances-worker.tf @@ -12,6 +12,7 @@ locals { node_name : zone cpu : lookup(try(var.instances[zone], {}), "worker_cpu", 1) mem : lookup(try(var.instances[zone], {}), "worker_mem", 2048) + ip0 : lookup(try(var.instances[zone], {}), "worke_ip0", "ip6=auto") ipv4 : "${cidrhost(local.subnets[zone], 4 + inx)}/24" gwv4 : local.gwv4 } @@ -28,12 +29,17 @@ resource "null_resource" "worker_machineconfig" { } provisioner "file" { - source = "${path.module}/_cfgs/worker.yaml" + content = templatefile("${path.module}/templates/worker.yaml.tpl", + merge(var.kubernetes, { + lbv4 = local.ipv4_vip + nodeSubnets = var.vpc_main_cidr + labels = local.worker_labels + })) destination = "/var/lib/vz/snippets/${local.worker_prefix}.yaml" } triggers = { - params = filemd5("${path.module}/_cfgs/worker.yaml") + params = filemd5("${path.module}/templates/worker.yaml.tpl") } } @@ -146,7 +152,7 @@ resource "proxmox_vm_qemu" "worker" { define_connection_info = false os_type = "ubuntu" qemu_os = "l26" - ipconfig0 = "ip6=auto" + ipconfig0 = each.value.ip0 ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "user=local:snippets/${local.worker_prefix}.yaml,meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage @@ -186,10 +192,19 @@ resource "proxmox_vm_qemu" "worker" { ssd = 1 backup = false } + disk { + type = "scsi" + storage = var.proxmox_storage + size = "128G" + cache = "none" + ssd = 1 + backup = false + } lifecycle { ignore_changes = [ boot, + disk, network, desc, numa, diff --git a/proxmox/outputs.tf b/proxmox/outputs.tf index ec3f490..ee306cb 100644 --- a/proxmox/outputs.tf +++ b/proxmox/outputs.tf @@ -4,6 +4,14 @@ output "controlplane_endpoint" { value = local.ipv4_vip } +output "controlplane_apply" { + description = "Kubernetes controlplane apply command" + value = [for cp in local.controlplanes : + "talosctl apply-config --insecure --nodes ${split("/", cp.ipv4)[0]} --config-patch @_cfgs/${cp.name}.yaml --file _cfgs/controlplane.yaml" + ] + depends_on = [proxmox_vm_qemu.controlplane] +} + output "controlplane_nodes" { description = "Kubernetes controlplane nodes" value = [ diff --git a/proxmox/templates/controlplane.yaml.tpl b/proxmox/templates/controlplane.yaml.tpl index 65ed33a..2680f8b 100644 --- a/proxmox/templates/controlplane.yaml.tpl +++ b/proxmox/templates/controlplane.yaml.tpl @@ -48,6 +48,8 @@ machine: allowedKubernetesNamespaces: - kube-system cluster: + adminKubeconfig: + certLifetime: 8h0m0s controlPlane: endpoint: https://${apiDomain}:6443 network: diff --git a/proxmox/templates/web.yaml.tpl b/proxmox/templates/web.yaml.tpl new file mode 100644 index 0000000..d840043 --- /dev/null +++ b/proxmox/templates/web.yaml.tpl @@ -0,0 +1,58 @@ +version: v1alpha1 +debug: false +persist: true +machine: + type: worker + token: ${tokenMachine} + ca: + crt: ${caMachine} + kubelet: + extraArgs: + cloud-provider: external + rotate-server-certificates: true + node-labels: "${labels}" + clusterDNS: + - 169.254.2.53 + - ${cidrhost(split(",",serviceSubnets)[0], 10)} + nodeIP: + validSubnets: ${format("%#v",split(",",nodeSubnets))} + network: + interfaces: + - interface: dummy0 + addresses: + - 169.254.2.53/32 + extraHostEntries: + - ip: ${lbv4} + aliases: + - ${apiDomain} + sysctls: + net.core.somaxconn: 65535 + net.core.netdev_max_backlog: 4096 + systemDiskEncryption: + state: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + ephemeral: + provider: luks2 + keys: + - nodeID: {} + slot: 0 + options: + - no_read_workqueue + - no_write_workqueue +cluster: + id: ${clusterID} + secret: ${clusterSecret} + controlPlane: + endpoint: https://${apiDomain}:6443 + clusterName: ${clusterName} + discovery: + enabled: true + network: + dnsDomain: ${domain} + serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + token: ${token} + ca: + crt: ${ca} diff --git a/proxmox/templates/worker.patch.yaml.tpl b/proxmox/templates/worker.patch.yaml.tpl new file mode 100644 index 0000000..db165ae --- /dev/null +++ b/proxmox/templates/worker.patch.yaml.tpl @@ -0,0 +1,45 @@ +machine: + kubelet: + extraArgs: + cloud-provider: external + rotate-server-certificates: true + node-labels: "project.io/node-pool=worker" + clusterDNS: + - 169.254.2.53 + - 10.200.0.10 + nodeIP: + validSubnets: ["172.16.0.0/24"] + network: + interfaces: + - interface: dummy0 + addresses: + - 169.254.2.53/32 + extraHostEntries: + - ip: 172.16.0.10 + aliases: + - api.cluster.local + sysctls: + net.core.somaxconn: 65535 + net.core.netdev_max_backlog: 4096 + systemDiskEncryption: + state: + provider: luks2 + options: + - no_read_workqueue + - no_write_workqueue + keys: + - nodeID: {} + slot: 0 + ephemeral: + provider: luks2 + options: + - no_read_workqueue + - no_write_workqueue + keys: + - nodeID: {} + slot: 0 +cluster: + controlPlane: + endpoint: https://api.cluster.local:6443 + proxy: + disabled: true diff --git a/proxmox/templates/worker.yaml.tpl b/proxmox/templates/worker.yaml.tpl index ca679c8..81e4527 100644 --- a/proxmox/templates/worker.yaml.tpl +++ b/proxmox/templates/worker.yaml.tpl @@ -1,8 +1,16 @@ +version: v1alpha1 +debug: false +persist: true machine: + type: worker + token: ${tokenMachine} + ca: + crt: ${caMachine} kubelet: extraArgs: cloud-provider: external rotate-server-certificates: true + node-labels: "${labels}" clusterDNS: - 169.254.2.53 - ${cidrhost(split(",",serviceSubnets)[0], 10)} @@ -23,22 +31,32 @@ machine: systemDiskEncryption: state: provider: luks2 - options: - - no_read_workqueue - - no_write_workqueue keys: - nodeID: {} slot: 0 ephemeral: provider: luks2 - options: - - no_read_workqueue - - no_write_workqueue keys: - nodeID: {} slot: 0 + options: + - no_read_workqueue + - no_write_workqueue + disks: + - device: /dev/sdb + partitions: + - mountpoint: /var/data cluster: + id: ${clusterID} + secret: ${clusterSecret} controlPlane: endpoint: https://${apiDomain}:6443 - proxy: - disabled: true + clusterName: ${clusterName} + discovery: + enabled: true + network: + dnsDomain: ${domain} + serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + token: ${token} + ca: + crt: ${ca} diff --git a/proxmox/variables.tf b/proxmox/variables.tf index 6e47fa1..9216f61 100644 --- a/proxmox/variables.tf +++ b/proxmox/variables.tf @@ -81,12 +81,14 @@ variable "controlplane" { count = 0, cpu = 2, mem = 4096, + # ip0 = "ip6=1:2::3/64,gw6=1:2::1" }, "node2" = { id = 510 count = 0, cpu = 2, mem = 4096, + # ip0 = "ip6=dhcp", } } } @@ -100,10 +102,12 @@ variable "instances" { web_count = 0, web_cpu = 2, web_mem = 4096, + web_ip0 = "", # ip=dhcp,ip6=dhcp worker_id = 1050 worker_count = 0, worker_cpu = 2, worker_mem = 4096, + worker_ip0 = "", # ip=dhcp,ip6=dhcp }, "node2" = { web_id = 2000 diff --git a/system_os/proxmox/proxmox.pkr.hcl b/system_os/proxmox/proxmox.pkr.hcl index 5f6fa7d..8b51c0f 100644 --- a/system_os/proxmox/proxmox.pkr.hcl +++ b/system_os/proxmox/proxmox.pkr.hcl @@ -42,9 +42,9 @@ source "proxmox" "talos" { cpu_type = "host" memory = 3072 - vga { - type = "serial0" - } + # vga { + # type = "serial0" + # } serials = ["socket"] ssh_username = "root"