From aea43c1c5ed036be668c40943e60f6162991f38b Mon Sep 17 00:00:00 2001 From: Serge Logvinov Date: Mon, 23 Oct 2023 23:09:14 +0300 Subject: [PATCH] upgrade proxmox setup --- proxmox/Makefile | 2 +- proxmox/deployments/local-path-storage.yaml | 140 -------------- proxmox/deployments/metrics-server.yaml | 197 -------------------- proxmox/deployments/pod-ephemeral.yaml | 44 +++++ proxmox/deployments/proxmox-csi.yaml | 41 ++++ proxmox/deployments/pv.yaml | 30 +++ proxmox/deployments/test-statefulset.yaml | 49 +++++ proxmox/images/.gitignore | 2 + proxmox/images/Makefile | 12 ++ proxmox/instances-controlplane.tf | 24 +-- proxmox/instances-web.tf | 24 +-- proxmox/instances-worker.tf | 25 +-- proxmox/network.tf | 2 +- proxmox/templates/controlplane.yaml.tpl | 4 +- proxmox/templates/web.yaml.tpl | 36 +++- proxmox/templates/worker.yaml.tpl | 40 +++- proxmox/variables.tf | 4 +- proxmox/versions.tf | 2 +- 18 files changed, 282 insertions(+), 396 deletions(-) delete mode 100644 proxmox/deployments/local-path-storage.yaml delete mode 100644 proxmox/deployments/metrics-server.yaml create mode 100644 proxmox/deployments/pod-ephemeral.yaml create mode 100644 proxmox/deployments/proxmox-csi.yaml create mode 100644 proxmox/deployments/pv.yaml create mode 100644 proxmox/deployments/test-statefulset.yaml create mode 100644 proxmox/images/.gitignore diff --git a/proxmox/Makefile b/proxmox/Makefile index 396afc0..24278fc 100644 --- a/proxmox/Makefile +++ b/proxmox/Makefile @@ -9,6 +9,7 @@ endif help: @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) +.PHONY: init init: ## Initialize terraform terraform init -upgrade @@ -20,7 +21,6 @@ create-config: ## Genereate talos configs create-templates: @echo 'podSubnets: "10.32.0.0/12,fd00:10:32::/102"' > _cfgs/tfstate.vars @echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"' >> _cfgs/tfstate.vars - @echo 'nodeSubnets: "172.16.0.0/12"' >> _cfgs/tfstate.vars @echo 'apiDomain: api.cluster.local' >> _cfgs/tfstate.vars @yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}' >> _cfgs/tfstate.vars @yq eval '.cluster.clusterName' _cfgs/controlplane.yaml | awk '{ print "clusterName: "$$1}' >> _cfgs/tfstate.vars diff --git a/proxmox/deployments/local-path-storage.yaml b/proxmox/deployments/local-path-storage.yaml deleted file mode 100644 index bfbd1c4..0000000 --- a/proxmox/deployments/local-path-storage.yaml +++ /dev/null @@ -1,140 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: local-path-storage - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: local-path-provisioner-service-account - namespace: local-path-storage - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: local-path-provisioner-role -rules: - - apiGroups: [ "" ] - resources: [ "nodes", "persistentvolumeclaims", "configmaps" ] - verbs: [ "get", "list", "watch" ] - - apiGroups: [ "" ] - resources: [ "endpoints", "persistentvolumes", "pods" ] - verbs: [ "*" ] - - apiGroups: [ "" ] - resources: [ "events" ] - verbs: [ "create", "patch" ] - - apiGroups: [ "storage.k8s.io" ] - resources: [ "storageclasses" ] - verbs: [ "get", "list", "watch" ] - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: local-path-provisioner-bind -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: local-path-provisioner-role -subjects: - - kind: ServiceAccount - name: local-path-provisioner-service-account - namespace: local-path-storage - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: local-path-provisioner - namespace: local-path-storage -spec: - replicas: 1 - selector: - matchLabels: - app: local-path-provisioner - template: - metadata: - labels: - app: local-path-provisioner - spec: - nodeSelector: - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - serviceAccountName: local-path-provisioner-service-account - containers: - - name: local-path-provisioner - image: rancher/local-path-provisioner:v0.0.23 - imagePullPolicy: IfNotPresent - command: - - local-path-provisioner - - --debug - - start - - --config - - /etc/config/config.json - volumeMounts: - - name: config-volume - mountPath: /etc/config/ - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumes: - - name: config-volume - configMap: - name: local-path-config - ---- -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: local-path - annotations: - storageclass.kubernetes.io/is-default-class: "true" -provisioner: rancher.io/local-path -volumeBindingMode: WaitForFirstConsumer -reclaimPolicy: Delete - ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: local-path-config - namespace: local-path-storage -data: - config.json: |- - { - "nodePathMap":[ - { - "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", - "paths":["/var/data"] - } - ] - } - setup: |- - #!/bin/sh - set -eu - mkdir -m 0777 -p "$VOL_DIR" - teardown: |- - #!/bin/sh - set -eu - rm -rf "$VOL_DIR" - helperPod.yaml: |- - apiVersion: v1 - kind: Pod - metadata: - name: helper-pod - spec: - priorityClassName: system-node-critical - tolerations: - - key: node.kubernetes.io/disk-pressure - operator: Exists - effect: NoSchedule - containers: - - name: helper-pod - image: busybox - imagePullPolicy: IfNotPresent diff --git a/proxmox/deployments/metrics-server.yaml b/proxmox/deployments/metrics-server.yaml deleted file mode 100644 index f259001..0000000 --- a/proxmox/deployments/metrics-server.yaml +++ /dev/null @@ -1,197 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - pods - - nodes - - nodes/stats - - namespaces - - configmaps - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - nodeSelector: - kubernetes.io/os: linux - node-role.kubernetes.io/control-plane: "" - tolerations: - - key: "node-role.kubernetes.io/control-plane" - effect: NoSchedule - containers: - - args: - - --cert-dir=/tmp - - --secure-port=6443 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - - --authorization-always-allow-paths=/metrics - image: k8s.gcr.io/metrics-server/metrics-server:v0.5.0 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 6443 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - volumeMounts: - - mountPath: /tmp - name: tmp-dir - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - volumes: - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io -spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100 diff --git a/proxmox/deployments/pod-ephemeral.yaml b/proxmox/deployments/pod-ephemeral.yaml new file mode 100644 index 0000000..ec7f290 --- /dev/null +++ b/proxmox/deployments/pod-ephemeral.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test + namespace: default +spec: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + # nodeSelector: + # kubernetes.io/hostname: kube-11 + containers: + - name: alpine + image: alpine + command: ["sleep","6000"] + volumeMounts: + - name: pvc + mountPath: /mnt + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + runAsNonRoot: true + terminationGracePeriodSeconds: 1 + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsUser: 65534 + volumes: + - name: pvc + ephemeral: + volumeClaimTemplate: + metadata: + labels: + type: pvc-volume + spec: + accessModes: [ "ReadWriteOnce" ] + storageClassName: proxmox-zfs + resources: + requests: + storage: 5Gi diff --git a/proxmox/deployments/proxmox-csi.yaml b/proxmox/deployments/proxmox-csi.yaml new file mode 100644 index 0000000..467ab80 --- /dev/null +++ b/proxmox/deployments/proxmox-csi.yaml @@ -0,0 +1,41 @@ + +controller: + plugin: + image: + pullPolicy: Always + tag: edge + +node: + plugin: + image: + pullPolicy: Always + tag: edge + + nodeSelector: + node.cloudprovider.kubernetes.io/platform: nocloud + tolerations: + - operator: Exists + +nodeSelector: + node-role.kubernetes.io/control-plane: "" +tolerations: + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + +config: + clusters: + - region: "dev-1" + token_id: "root@pam!terraform" + token_secret: "cb6e5561-ce10-4e7e-8b99-155ff6371a48" + url: "https://192.168.10.4:8006/api2/json" + insecure: true + +storageClass: + - name: proxmox + storage: local-lvm + reclaimPolicy: Delete + fstype: xfs + - name: proxmox-zfs + storage: zfs + reclaimPolicy: Delete + fstype: xfs diff --git a/proxmox/deployments/pv.yaml b/proxmox/deployments/pv.yaml new file mode 100644 index 0000000..d5e7a28 --- /dev/null +++ b/proxmox/deployments/pv.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: pvc-test +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + csi: + driver: csi.proxmox.sinextra.dev + fsType: xfs + volumeAttributes: + storage: zfs + volumeHandle: dev-1/pve-m-4/zfs/vm-9999-pvc-test + storageClassName: proxmox-zfs +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: storage-test-0 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: proxmox-zfs + volumeName: pvc-test diff --git a/proxmox/deployments/test-statefulset.yaml b/proxmox/deployments/test-statefulset.yaml new file mode 100644 index 0000000..be27f8c --- /dev/null +++ b/proxmox/deployments/test-statefulset.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test + namespace: kube-system + labels: + app: alpine +spec: + podManagementPolicy: Parallel # default is OrderedReady + serviceName: test + replicas: 1 + template: + metadata: + labels: + app: alpine + spec: + terminationGracePeriodSeconds: 3 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + nodeSelector: + # kubernetes.io/hostname: kube-21 + # topology.kubernetes.io/zone: hvm-1 + containers: + - name: alpine + image: alpine + command: ["sleep","1d"] + securityContext: + seccompProfile: + type: RuntimeDefault + capabilities: + drop: ["ALL"] + volumeMounts: + - name: storage + mountPath: /mnt + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app: alpine + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 5Gi + storageClassName: proxmox-zfs diff --git a/proxmox/images/.gitignore b/proxmox/images/.gitignore new file mode 100644 index 0000000..aa3bb53 --- /dev/null +++ b/proxmox/images/.gitignore @@ -0,0 +1,2 @@ +*.qcow2.xz +*.qcow2 diff --git a/proxmox/images/Makefile b/proxmox/images/Makefile index 1b8aed3..30f0add 100644 --- a/proxmox/images/Makefile +++ b/proxmox/images/Makefile @@ -1,6 +1,18 @@ +# +REGISTRY ?= ghcr.io/siderolabs +TAG ?= 1.5.4 + +clean: + rm -f nocloud-*.qcow2.xz + rm -f nocloud-*.qcow2 init: packer init -upgrade . release: packer build -only=release.proxmox.talos . + +images: clean + docker run --rm -i -v /dev:/dev --privileged $(REGISTRY)/imager:v$(TAG) oracle \ + --extra-kernel-arg talos.dashboard.disabled=1 --platform nocloud --arch amd64 --tar-to-stdout | tar xz + xz -d nocloud-amd64.qcow2.xz diff --git a/proxmox/instances-controlplane.tf b/proxmox/instances-controlplane.tf index c06c1e2..0ffebd5 100644 --- a/proxmox/instances-controlplane.tf +++ b/proxmox/instances-controlplane.tf @@ -50,12 +50,12 @@ resource "proxmox_vm_qemu" "controlplane" { target_node = each.value.node_name clone = var.proxmox_image - agent = 0 - define_connection_info = false - os_type = "ubuntu" - qemu_os = "l26" - ipconfig0 = each.value.ip0 - ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" + agent = 0 + define_connection_info = false + os_type = "ubuntu" + qemu_os = "l26" + # ipconfig1 = each.value.ip0 + ipconfig0 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage @@ -75,15 +75,15 @@ resource "proxmox_vm_qemu" "controlplane" { type = "socket" } - network { - model = "virtio" - bridge = "vmbr0" - firewall = true - } network { model = "virtio" - bridge = "vmbr1" + bridge = "vmbr0" + # firewall = true } + # network { + # model = "virtio" + # bridge = "vmbr1" + # } boot = "order=scsi0" disk { diff --git a/proxmox/instances-web.tf b/proxmox/instances-web.tf index 65f4355..199076d 100644 --- a/proxmox/instances-web.tf +++ b/proxmox/instances-web.tf @@ -31,9 +31,10 @@ resource "null_resource" "web_machineconfig" { provisioner "file" { # source = "${path.module}/_cfgs/worker.yaml" content = templatefile("${path.module}/templates/web.yaml.tpl", - merge(var.kubernetes, { + merge(var.kubernetes, try(var.instances["all"], {}), { lbv4 = local.ipv4_vip nodeSubnets = var.vpc_main_cidr + clusterDns = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10) labels = local.web_labels })) @@ -77,12 +78,12 @@ resource "proxmox_vm_qemu" "web" { target_node = each.value.node_name clone = var.proxmox_image - agent = 0 - define_connection_info = false - os_type = "ubuntu" - qemu_os = "l26" - ipconfig0 = each.value.ip0 - ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" + agent = 0 + define_connection_info = false + os_type = "ubuntu" + qemu_os = "l26" + # ipconfig0 = each.value.ip0 + ipconfig0 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "user=local:snippets/${local.web_prefix}.yaml,meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage @@ -91,6 +92,7 @@ resource "proxmox_vm_qemu" "web" { sockets = 1 cores = each.value.cpu memory = each.value.mem + numa = true scsihw = "virtio-scsi-single" vga { @@ -107,10 +109,10 @@ resource "proxmox_vm_qemu" "web" { bridge = "vmbr0" firewall = true } - network { - model = "virtio" - bridge = "vmbr1" - } + # network { + # model = "virtio" + # bridge = "vmbr1" + # } boot = "order=scsi0" disk { diff --git a/proxmox/instances-worker.tf b/proxmox/instances-worker.tf index ed5eb97..e2bc572 100644 --- a/proxmox/instances-worker.tf +++ b/proxmox/instances-worker.tf @@ -30,9 +30,10 @@ resource "null_resource" "worker_machineconfig" { provisioner "file" { content = templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, { + merge(var.kubernetes, try(var.instances["all"], {}), { lbv4 = local.ipv4_vip nodeSubnets = var.vpc_main_cidr + clusterDns = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10) labels = local.worker_labels })) destination = "/var/lib/vz/snippets/${local.worker_prefix}.yaml" @@ -149,12 +150,12 @@ resource "proxmox_vm_qemu" "worker" { target_node = each.value.node_name clone = var.proxmox_image - agent = 0 - define_connection_info = false - os_type = "ubuntu" - qemu_os = "l26" - ipconfig0 = each.value.ip0 - ipconfig1 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" + agent = 0 + define_connection_info = false + os_type = "ubuntu" + qemu_os = "l26" + # ipconfig0 = each.value.ip0 + ipconfig0 = "ip=${each.value.ipv4},gw=${each.value.gwv4}" cicustom = "user=local:snippets/${local.worker_prefix}.yaml,meta=local:snippets/${each.value.name}.metadata.yaml" cloudinit_cdrom_storage = var.proxmox_storage @@ -163,6 +164,7 @@ resource "proxmox_vm_qemu" "worker" { sockets = 1 cores = each.value.cpu memory = each.value.mem + numa = true scsihw = "virtio-scsi-single" vga { @@ -193,15 +195,6 @@ resource "proxmox_vm_qemu" "worker" { ssd = 1 backup = false } - disk { - type = "scsi" - storage = var.proxmox_storage - size = "128G" - cache = "none" - iothread = 1 - ssd = 1 - backup = false - } lifecycle { ignore_changes = [ diff --git a/proxmox/network.tf b/proxmox/network.tf index 099c861..e3bb67c 100644 --- a/proxmox/network.tf +++ b/proxmox/network.tf @@ -1,6 +1,6 @@ locals { - zones = [for k, v in var.instances : k] + zones = [for k, v in var.instances : k if k != "all"] controlplane_subnet = cidrsubnet(var.vpc_main_cidr, 5, var.network_shift) subnets = { for inx, zone in local.zones : zone => cidrsubnet(var.vpc_main_cidr, 5, var.network_shift + inx + 1) } diff --git a/proxmox/templates/controlplane.yaml.tpl b/proxmox/templates/controlplane.yaml.tpl index b29ca6e..2e2c6f7 100644 --- a/proxmox/templates/controlplane.yaml.tpl +++ b/proxmox/templates/controlplane.yaml.tpl @@ -10,7 +10,7 @@ machine: network: hostname: "${name}" interfaces: - - interface: eth1 + - interface: eth0 vip: ip: ${ipv4_vip} - interface: dummy0 @@ -49,7 +49,7 @@ machine: - kube-system cluster: adminKubeconfig: - certLifetime: 8h0m0s + certLifetime: 48h0m0s controlPlane: endpoint: https://${apiDomain}:6443 network: diff --git a/proxmox/templates/web.yaml.tpl b/proxmox/templates/web.yaml.tpl index d840043..6e337da 100644 --- a/proxmox/templates/web.yaml.tpl +++ b/proxmox/templates/web.yaml.tpl @@ -7,13 +7,15 @@ machine: ca: crt: ${caMachine} kubelet: + image: ghcr.io/siderolabs/kubelet:${version} + defaultRuntimeSeccompProfileEnabled: true extraArgs: cloud-provider: external rotate-server-certificates: true - node-labels: "${labels}" + node-labels: ${labels} clusterDNS: - 169.254.2.53 - - ${cidrhost(split(",",serviceSubnets)[0], 10)} + - ${clusterDns} nodeIP: validSubnets: ${format("%#v",split(",",nodeSubnets))} network: @@ -25,23 +27,45 @@ machine: - ip: ${lbv4} aliases: - ${apiDomain} + nameservers: + - 2606:4700:4700::1111 + - 1.1.1.1 + - 2001:4860:4860::8888 + time: + servers: + - 2.europe.pool.ntp.org + - time.cloudflare.com sysctls: net.core.somaxconn: 65535 net.core.netdev_max_backlog: 4096 + net.ipv4.tcp_keepalive_intvl: 60 + net.ipv4.tcp_keepalive_time: 600 + vm.max_map_count: 128000 + install: + wipe: true + extraKernelArgs: + - talos.dashboard.disabled=1 systemDiskEncryption: state: provider: luks2 + options: + - no_read_workqueue + - no_write_workqueue keys: - nodeID: {} slot: 0 ephemeral: provider: luks2 - keys: - - nodeID: {} - slot: 0 options: - no_read_workqueue - no_write_workqueue + keys: + - nodeID: {} + slot: 0 + features: + rbac: true + stableHostname: true + apidCheckExtKeyUsage: true cluster: id: ${clusterID} secret: ${clusterSecret} @@ -53,6 +77,8 @@ cluster: network: dnsDomain: ${domain} serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + proxy: + disabled: false token: ${token} ca: crt: ${ca} diff --git a/proxmox/templates/worker.yaml.tpl b/proxmox/templates/worker.yaml.tpl index 81e4527..6e337da 100644 --- a/proxmox/templates/worker.yaml.tpl +++ b/proxmox/templates/worker.yaml.tpl @@ -7,13 +7,15 @@ machine: ca: crt: ${caMachine} kubelet: + image: ghcr.io/siderolabs/kubelet:${version} + defaultRuntimeSeccompProfileEnabled: true extraArgs: cloud-provider: external rotate-server-certificates: true - node-labels: "${labels}" + node-labels: ${labels} clusterDNS: - 169.254.2.53 - - ${cidrhost(split(",",serviceSubnets)[0], 10)} + - ${clusterDns} nodeIP: validSubnets: ${format("%#v",split(",",nodeSubnets))} network: @@ -25,27 +27,45 @@ machine: - ip: ${lbv4} aliases: - ${apiDomain} + nameservers: + - 2606:4700:4700::1111 + - 1.1.1.1 + - 2001:4860:4860::8888 + time: + servers: + - 2.europe.pool.ntp.org + - time.cloudflare.com sysctls: net.core.somaxconn: 65535 net.core.netdev_max_backlog: 4096 + net.ipv4.tcp_keepalive_intvl: 60 + net.ipv4.tcp_keepalive_time: 600 + vm.max_map_count: 128000 + install: + wipe: true + extraKernelArgs: + - talos.dashboard.disabled=1 systemDiskEncryption: state: provider: luks2 + options: + - no_read_workqueue + - no_write_workqueue keys: - nodeID: {} slot: 0 ephemeral: provider: luks2 - keys: - - nodeID: {} - slot: 0 options: - no_read_workqueue - no_write_workqueue - disks: - - device: /dev/sdb - partitions: - - mountpoint: /var/data + keys: + - nodeID: {} + slot: 0 + features: + rbac: true + stableHostname: true + apidCheckExtKeyUsage: true cluster: id: ${clusterID} secret: ${clusterSecret} @@ -57,6 +77,8 @@ cluster: network: dnsDomain: ${domain} serviceSubnets: ${format("%#v",split(",",serviceSubnets))} + proxy: + disabled: false token: ${token} ca: crt: ${ca} diff --git a/proxmox/variables.tf b/proxmox/variables.tf index f7ee43a..f65ff74 100644 --- a/proxmox/variables.tf +++ b/proxmox/variables.tf @@ -48,7 +48,6 @@ variable "kubernetes" { default = { podSubnets = "10.32.0.0/12,fd40:10:32::/102" serviceSubnets = "10.200.0.0/22,fd40:10:200::/112" - nodeSubnets = "192.168.0.0/16" domain = "cluster.local" apiDomain = "api.cluster.local" clusterName = "talos-k8s-proxmox" @@ -97,6 +96,9 @@ variable "instances" { description = "Map of VMs launched on proxmox hosts" type = map(any) default = { + "all" = { + version = "v1.28.2" + }, "node1" = { web_id = 1000 web_count = 0, diff --git a/proxmox/versions.tf b/proxmox/versions.tf index ddbd2ab..cdf21f3 100644 --- a/proxmox/versions.tf +++ b/proxmox/versions.tf @@ -6,7 +6,7 @@ terraform { } # proxmox = { # source = "bpg/proxmox" - # version = "0.17.0-rc1" + # version = "~> 0.35.1" # } } required_version = ">= 1.0"