diff --git a/README.md b/README.md index b2cd657..54d77ad 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,11 @@ Having a single Kubernetes control plane that spans multiple cloud providers can | [Azure](azure) | 1.3.4 | CCM,CSI,Autoscaler | many regions, many zones | ✓ | ✓ | | [Exoscale](exoscale) | 1.3.0 | CCM,Autoscaler | many regions | ✗ | | | [GCP](gcp-zonal) | 1.3.4 | CCM,CSI,Autoscaler | one region, many zones | ✓ | ✓ | -| [Hetzner](hetzner) | 1.4.0 | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ | +| [Hetzner](hetzner) | 1.8.0 | CCM,CSI,Autoscaler | many regions, one network zone | ✗ | ✓ | | [Openstack](openstack) | 1.3.4 | CCM,CSI | many regions, many zones | ✓ | ✓ | -| [Oracle](oracle) | 1.3.4 | CCM,~~CSI~~,Autoscaler | one region, many zones | ✓ | ✓ | -| [Proxmox](proxmox) | 1.3.4 | CCM,CSI | one region, mny zones | ✓ | ✓ | -| [Scaleway](scaleway) | 1.3.4 | CCM,CSI | one region | ✓ | ✓ | +| [Oracle](oracle) | 1.3.4 | CCM,CSI,Autoscaler | one region, many zones | ✓ | ✓ | +| [Proxmox](proxmox) | 1.8.0 | CCM,CSI | one region, mny zones | ✓ | ✓ | +| [Scaleway](scaleway) | 1.8.0 | CCM,CSI | one region | ✓ | ✓ | ## Known issues diff --git a/hetzner/.gitignore b/hetzner/.gitignore index 64eb163..0872560 100644 --- a/hetzner/.gitignore +++ b/hetzner/.gitignore @@ -1,5 +1,10 @@ _cfgs/ -templates/controlplane.yaml -controlplane-*.yaml -worker-*.yaml -*.patch +.terraform.lock.hcl +.terraform.tfstate.lock.info +/terraform.tfstate +terraform.tfstate.backup +terraform.tfvars +terraform.tfvars.json +terraform.tfvars.sops.json +# +age.key.txt diff --git a/hetzner/.sops.yaml b/hetzner/.sops.yaml new file mode 100644 index 0000000..dfc10bd --- /dev/null +++ b/hetzner/.sops.yaml @@ -0,0 +1,21 @@ +--- +creation_rules: + - path_regex: \.env\.yaml$ + key_groups: + - age: + - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf + - path_regex: terraform\.tfvars\.sops\.json$ + encrypted_regex: "(token|Secret|ID)" + key_groups: + - age: + - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf + - path_regex: _cfgs/controlplane.yaml$ + encrypted_regex: "(token|key|secret|id)" + key_groups: + - age: + - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf + - path_regex: _cfgs/talosconfig$ + encrypted_regex: "key" + key_groups: + - age: + - age1ngvggfld4elq68926uczkes9rcqfjhnqn0tr6l8avyp4h46qzucqvx3sdf diff --git a/hetzner/Makefile b/hetzner/Makefile index 9871174..4b6493d 100644 --- a/hetzner/Makefile +++ b/hetzner/Makefile @@ -1,9 +1,11 @@ CLUSTERNAME := "talos-k8s-hetzner" CPFIRST := ${shell terraform output -raw controlplane_firstnode 2>/dev/null} -ENDPOINT := ${shell terraform output -raw controlplane_endpoint 2>/dev/null} -ifneq (,$(findstring Warning,${ENDPOINT})) -ENDPOINT := api.cluster.local +ENDPOINT := ${shell terraform output -raw controlplane_firstnode 2>/dev/null} +ifeq ($(ENDPOINT),) +ENDPOINT := 127.0.0.1 +else ifneq (,$(findstring Warning,${ENDPOINT})) +ENDPOINT := 127.0.0.1 endif help: @@ -11,23 +13,18 @@ help: clean: ## Clean all terraform destroy -auto-approve - rm -rf _cfgs - rm -f kubeconfig terraform.tfvars.json + rm -rf .terraform.lock.hcl .terraform/ terraform.tfstate terraform.tfstate.backup + rm -f kubeconfig terraform.tfvars.sops.json prepare: @[ -f ~/.ssh/terraform ] || ssh-keygen -f ~/.ssh/terraform -N '' -t rsa -create-lb: ## Create load balancer - terraform init - terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api - terraform refresh - create-config: ## Genereate talos configs talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false ${CLUSTERNAME} https://${ENDPOINT}:6443 talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT} create-templates: - @echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/102"' > _cfgs/tfstate.vars + @echo 'podSubnets: "10.32.0.0/12,fd40:10:32::/96"' > _cfgs/tfstate.vars @echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"' >> _cfgs/tfstate.vars @echo 'apiDomain: api.cluster.local' >> _cfgs/tfstate.vars @yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}' >> _cfgs/tfstate.vars @@ -39,36 +36,66 @@ create-templates: @yq eval '.cluster.token' _cfgs/controlplane.yaml | awk '{ print "token: "$$1}' >> _cfgs/tfstate.vars @yq eval '.cluster.ca.crt' _cfgs/controlplane.yaml | awk '{ print "ca: "$$1}' >> _cfgs/tfstate.vars - @yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json + @yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.sops.json + @sops --encrypt -i terraform.tfvars.sops.json + @yq eval .ca _cfgs/tfstate.vars | base64 --decode > _cfgs/ca.crt + @sops --encrypt --input-type=yaml --output-type=yaml _cfgs/talosconfig > _cfgs/talosconfig.sops.yaml + @sops --encrypt --input-type=yaml --output-type=yaml _cfgs/controlplane.yaml > _cfgs/controlplane.sops.yaml -create-controlplane-bootstrap: - talosctl --talosconfig _cfgs/talosconfig config endpoint ${CPFIRST} - talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} bootstrap - -create-controlplane: ## Bootstrap first controlplane node - terraform apply -auto-approve -target=hcloud_server.controlplane -target=null_resource.controlplane +create-lb: ## Create load balancer + terraform init + terraform apply -auto-approve -target=hcloud_floating_ip.api -target=hcloud_load_balancer.api + terraform refresh create-infrastructure: ## Bootstrap all nodes terraform apply -create-kubeconfig: ## Prepare kubeconfig - talosctl --talosconfig _cfgs/talosconfig --nodes ${CPFIRST} kubeconfig . - kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://${ENDPOINT}:6443 - kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system +bootstrap: ## Bootstrap controlplane + talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT} + talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} bootstrap -create-secrets: - dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret - kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret - rm -f hcloud-csi-secret.secret +.PHONY: kubeconfig +kubeconfig: ## Download kubeconfig + rm -f kubeconfig + talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT} + talosctl --talosconfig _cfgs/talosconfig --nodes ${ENDPOINT} kubeconfig . + kubectl --kubeconfig=kubeconfig config set clusters.${CLUSTERNAME}.server https://[${ENDPOINT}]:6443 + kubectl --kubeconfig=kubeconfig config set-context --current --namespace=kube-system helm-repos: ## add helm repos helm repo add hcloud https://charts.hetzner.cloud helm repo add autoscaler https://kubernetes.github.io/autoscaler helm repo update -create-deployments: +system-static: + helm template --namespace=kube-system -f deployments/talos-ccm.yaml \ + --set useDaemonSet=true \ + talos-cloud-controller-manager \ + oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager > deployments/talos-cloud-controller-manager-result.yaml + helm template --namespace=kube-system -f deployments/hcloud-ccm.yaml \ hcloud-cloud-controller-manager hcloud/hcloud-cloud-controller-manager > deployments/hcloud-cloud-controller-manager-result.yaml - helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \ - autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml + # helm template --namespace=kube-system -f deployments/hcloud-autoscaler.yaml cluster-autoscaler-hcloud \ + # autoscaler/cluster-autoscaler > deployments/hcloud-autoscaler-result.yaml + +system: + helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system --version=1.15.7 -f deployments/cilium.yaml \ + cilium cilium/cilium + + kubectl --kubeconfig=kubeconfig -n kube-system delete svc cilium-agent + + kubectl --kubeconfig=kubeconfig apply -f ../_deployments/vars/coredns-local.yaml + + helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f ../_deployments/vars/metrics-server.yaml \ + metrics-server metrics-server/metrics-server + + helm --kubeconfig=kubeconfig upgrade -i --namespace=kube-system -f deployments/talos-ccm.yaml \ + --set useDaemonSet=true \ + talos-cloud-controller-manager \ + oci://ghcr.io/siderolabs/charts/talos-cloud-controller-manager + +deploy-csi: + dd if=/dev/urandom bs=1 count=16 2>/dev/null | hexdump -e '"%00x"' > hcloud-csi-secret.secret + kubectl --kubeconfig=kubeconfig create secret generic hcloud-csi-secret --from-file=encryptionPassphrase=hcloud-csi-secret.secret + rm -f hcloud-csi-secret.secret diff --git a/hetzner/common.tf b/hetzner/common.tf index 87137e1..4a2bf90 100644 --- a/hetzner/common.tf +++ b/hetzner/common.tf @@ -1,6 +1,6 @@ data "hcloud_image" "talos" { - for_each = toset(["amd64", "arm64"]) + for_each = toset(var.arch) with_architecture = each.key == "amd64" ? "x86" : "arm" with_selector = "type=infra" } diff --git a/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml index 20b8d3d..67b2228 100644 --- a/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml +++ b/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml @@ -61,8 +61,7 @@ spec: effect: "NoExecute" containers: - name: hcloud-cloud-controller-manager - command: - - "/bin/hcloud-cloud-controller-manager" + args: - "--allow-untagged-cloud" - "--cloud-provider=hcloud" - "--route-reconciliation-period=30s" @@ -74,11 +73,19 @@ spec: secretKeyRef: key: token name: hcloud - - name: NODE_NAME + - name: ROBOT_PASSWORD valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: hetznercloud/hcloud-cloud-controller-manager:v1.17.2 # x-release-please-version + secretKeyRef: + key: robot-password + name: hcloud + optional: true + - name: ROBOT_USER + valueFrom: + secretKeyRef: + key: robot-user + name: hcloud + optional: true + image: docker.io/hetznercloud/hcloud-cloud-controller-manager:v1.20.0 # x-release-please-version ports: - name: metrics containerPort: 8233 diff --git a/hetzner/deployments/talos-ccm.yaml b/hetzner/deployments/talos-ccm.yaml new file mode 100644 index 0000000..aed679f --- /dev/null +++ b/hetzner/deployments/talos-ccm.yaml @@ -0,0 +1,56 @@ + +image: + # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager + tag: edge + +service: + containerPort: 50258 + annotations: + prometheus.io/scrape: "true" + prometheus.io/scheme: "https" + prometheus.io/port: "50258" + +logVerbosityLevel: 4 + +enabledControllers: + - cloud-node + # - node-ipam-controller + +# extraArgs: +# - --allocate-node-cidrs +# - --cidr-allocator-type=CloudAllocator +# - --node-cidr-mask-size-ipv4=24 +# - --node-cidr-mask-size-ipv6=80 + +tolerations: + - effect: NoSchedule + operator: Exists + +transformations: + - name: web + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^web-.+$ + labels: + node-role.kubernetes.io/web: "" + - name: worker + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^worker-.+$ + labels: + node-role.kubernetes.io/worker: "" + - name: db + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^db-.+$ + labels: + node-role.kubernetes.io/db: "" diff --git a/hetzner/deployments/talos-cloud-controller-manager-result.yaml b/hetzner/deployments/talos-cloud-controller-manager-result.yaml new file mode 100644 index 0000000..b840af6 --- /dev/null +++ b/hetzner/deployments/talos-cloud-controller-manager-result.yaml @@ -0,0 +1,318 @@ +--- +# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/serviceaccount.yaml +apiVersion: talos.dev/v1alpha1 +kind: ServiceAccount +metadata: + name: talos-cloud-controller-manager-talos-secrets + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + roles: + - os:reader +--- +# Source: talos-cloud-controller-manager/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +data: + ccm-config.yaml: | + global: + approveNodeCSR: true + transformations: + - labels: + node-role.kubernetes.io/web: "" + name: web + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^web-.+$ + - labels: + node-role.kubernetes.io/worker: "" + name: worker + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^worker-.+$ + - labels: + node-role.kubernetes.io/db: "" + name: db + nodeSelector: + - matchExpressions: + - key: hostname + operator: Regexp + values: + - ^db-.+$ +--- +# Source: talos-cloud-controller-manager/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - create + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - get +- apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - kubernetes.io/kubelet-serving + verbs: + - approve +--- +# Source: talos-cloud-controller-manager/templates/rolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:talos-cloud-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:talos-cloud-controller-manager +subjects: +- kind: ServiceAccount + name: talos-cloud-controller-manager + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: system:talos-cloud-controller-manager:extension-apiserver-authentication-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: + - kind: ServiceAccount + name: talos-cloud-controller-manager + namespace: kube-system +--- +# Source: talos-cloud-controller-manager/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm + annotations: + prometheus.io/port: "50258" + prometheus.io/scheme: https + prometheus.io/scrape: "true" + namespace: kube-system +spec: + clusterIP: None + type: ClusterIP + ports: + - name: https + port: 50258 + targetPort: 50258 + protocol: TCP + selector: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager +--- +# Source: talos-cloud-controller-manager/templates/deployment.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: talos-cloud-controller-manager + labels: + helm.sh/chart: talos-cloud-controller-manager-0.3.1 + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + app.kubernetes.io/version: "v1.6.0" + app.kubernetes.io/managed-by: Helm + namespace: kube-system +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + template: + metadata: + labels: + app.kubernetes.io/name: talos-cloud-controller-manager + app.kubernetes.io/instance: talos-cloud-controller-manager + spec: + serviceAccountName: talos-cloud-controller-manager + securityContext: + fsGroup: 10258 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 10258 + runAsNonRoot: true + runAsUser: 10258 + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + priorityClassName: system-cluster-critical + containers: + - name: talos-cloud-controller-manager + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + image: "ghcr.io/siderolabs/talos-cloud-controller-manager:edge" + imagePullPolicy: IfNotPresent + command: ["/talos-cloud-controller-manager"] + args: + - --v=4 + - --cloud-provider=talos + - --cloud-config=/etc/talos/ccm-config.yaml + - --controllers=cloud-node + - --leader-elect-resource-name=cloud-controller-manager-talos + - --use-service-account-credentials + - --secure-port=50258 + - --authorization-always-allow-paths=/healthz,/livez,/readyz,/metrics + env: + - name: TALOS_ENDPOINTS + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_HOST + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: KUBERNETES_SERVICE_PORT + value: "6443" + ports: + - containerPort: 50258 + name: https + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: https + scheme: HTTPS + initialDelaySeconds: 20 + periodSeconds: 30 + timeoutSeconds: 5 + resources: + requests: + cpu: 10m + memory: 64Mi + volumeMounts: + - name: cloud-config + mountPath: /etc/talos + readOnly: true + - name: talos-secrets + mountPath: /var/run/secrets/talos.dev + readOnly: true + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoSchedule + key: node.kubernetes.io/not-ready + operator: Exists + volumes: + - name: cloud-config + configMap: + name: talos-cloud-controller-manager + defaultMode: 416 # 0640 + - name: talos-secrets + secret: + secretName: talos-cloud-controller-manager-talos-secrets + defaultMode: 416 # 0640 diff --git a/hetzner/images/hetzner.pkr.hcl b/hetzner/images/hetzner.pkr.hcl index 3ebae0e..59c4e0e 100644 --- a/hetzner/images/hetzner.pkr.hcl +++ b/hetzner/images/hetzner.pkr.hcl @@ -2,8 +2,8 @@ packer { required_plugins { hcloud = { - version = ">= 1.0.5" - source = "github.com/hashicorp/hcloud" + version = ">= 1.5.0" + source = "github.com/hetznercloud/hcloud" } } } @@ -11,7 +11,7 @@ packer { source "hcloud" "talos" { token = var.hcloud_token rescue = "linux64" - image = "debian-11" + image = "debian-12" location = var.hcloud_location server_type = var.hcloud_type diff --git a/hetzner/images/variables.pkr.hcl b/hetzner/images/variables.pkr.hcl index ae155fa..16d5b13 100644 --- a/hetzner/images/variables.pkr.hcl +++ b/hetzner/images/variables.pkr.hcl @@ -12,12 +12,12 @@ variable "hcloud_location" { variable "hcloud_type" { type = string - default = "cx11" # cx11|cax11 (arm) + default = "cax11" # cx11|cax11 (arm) } variable "talos_version" { type = string - default = "v1.4.1" + default = "v1.7.6" } locals { diff --git a/hetzner/instances-controlplane.tf b/hetzner/instances-controlplane.tf index 9163f73..a450ff1 100644 --- a/hetzner/instances-controlplane.tf +++ b/hetzner/instances-controlplane.tf @@ -35,22 +35,6 @@ resource "hcloud_server" "controlplane" { ip = each.value.ip } - # user_data = templatefile("${path.module}/templates/controlplane.yaml", - # merge(var.kubernetes, { - # name = each.value.name - # ipv4_vip = local.ipv4_vip - # ipv4_local = each.value.ip - # lbv4_local = local.lbv4_local - # lbv4 = local.lbv4 - # lbv6 = local.lbv6 - # hcloud_network = hcloud_network.main.id - # hcloud_token = var.hcloud_token - # hcloud_image = data.hcloud_image.talos["amd64"].id - # robot_user = var.robot_user - # robot_password = var.robot_password - # }) - # ) - lifecycle { ignore_changes = [ network, @@ -73,38 +57,34 @@ resource "hcloud_load_balancer_target" "api" { # Secure push talos config to the controlplane # -resource "local_file" "controlplane" { +resource "local_sensitive_file" "controlplane" { for_each = local.controlplanes - content = templatefile("${path.module}/templates/controlplane.yaml.tpl", - { - name = each.value.name - apiDomain = var.kubernetes["apiDomain"] - domain = var.kubernetes["domain"] - podSubnets = var.kubernetes["podSubnets"] - serviceSubnets = var.kubernetes["serviceSubnets"] - ipv4_vip = local.ipv4_vip - ipv4_local = each.value.ip - lbv4_local = local.lbv4_local - lbv4 = local.lbv4 - lbv6 = local.lbv6 - nodeSubnets = hcloud_network_subnet.core.ip_range + merge(local.kubernetes, try(var.instances["all"], {}), { + name = each.value.name + nodeSubnets = hcloud_network_subnet.core.ip_range + ipv4_vip = local.ipv4_vip + ipv4_local = each.value.ip + lbv4_local = local.lbv4_local + lbv4 = local.lbv4 + lbv6 = local.lbv6 + hcloud_network = hcloud_network.main.id hcloud_token = var.hcloud_token hcloud_image = data.hcloud_image.talos["amd64"].id hcloud_sshkey = hcloud_ssh_key.infra.id robot_user = var.robot_user robot_password = var.robot_password - } + }) ) filename = "_cfgs/${each.value.name}.yaml" file_permission = "0600" } -resource "null_resource" "controlplane" { - for_each = local.controlplanes - provisioner "local-exec" { - command = "sleep 30 && talosctl apply-config --insecure --nodes ${hcloud_server.controlplane[each.key].ipv4_address} --timeout 5m0s --config-patch @_cfgs/${each.value.name}.yaml --file _cfgs/controlplane.yaml" - } - depends_on = [hcloud_load_balancer_target.api, local_file.controlplane] +locals { + controlplane_config = { for k, v in local.controlplanes : v.name => "talosctl apply-config --insecure --nodes ${hcloud_server.controlplane[k].ipv4_address} --config-patch @_cfgs/${v.name}.yaml --file _cfgs/controlplane.yaml" } +} + +output "controlplane_config" { + value = local.controlplane_config } diff --git a/hetzner/templates/controlplane.yaml.tpl b/hetzner/templates/controlplane.yaml.tpl index 36e8bea..3d6b79c 100644 --- a/hetzner/templates/controlplane.yaml.tpl +++ b/hetzner/templates/controlplane.yaml.tpl @@ -7,6 +7,7 @@ machine: - "${ipv4_vip}" - "${apiDomain}" kubelet: + image: ghcr.io/siderolabs/kubelet:${version} extraArgs: rotate-server-certificates: true clusterDNS: @@ -15,7 +16,7 @@ machine: nodeIP: validSubnets: ${format("%#v",split(",",nodeSubnets))} network: - hostname: "${name}" + hostname: ${name} interfaces: - interface: eth0 dhcp: true @@ -65,9 +66,11 @@ machine: - kube-system cluster: adminKubeconfig: - certLifetime: 8h0m0s + certLifetime: 48h0m0s controlPlane: endpoint: https://${apiDomain}:6443 + discovery: + enabled: false network: dnsDomain: ${domain} podSubnets: ${format("%#v",split(",",podSubnets))} @@ -79,6 +82,7 @@ cluster: proxy: disabled: true apiServer: + image: registry.k8s.io/kube-apiserver:${version} certSANs: - "${lbv4}" - "${lbv6}" @@ -87,9 +91,12 @@ cluster: - "${ipv4_vip}" - "${apiDomain}" controllerManager: + image: registry.k8s.io/kube-controller-manager:${version} extraArgs: - node-cidr-mask-size-ipv4: 24 - node-cidr-mask-size-ipv6: 112 + node-cidr-mask-size-ipv4: "24" + node-cidr-mask-size-ipv6: "112" + scheduler: + image: registry.k8s.io/kube-scheduler:${version} etcd: advertisedSubnets: - ${nodeSubnets} @@ -114,10 +121,8 @@ cluster: externalCloudProvider: enabled: true manifests: - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/talos-cloud-controller-manager-result.yaml - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager.yaml - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-csi.yaml - - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/metrics-server-result.yaml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/talos-cloud-controller-manager-result.yaml + - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/hetzner/deployments/hcloud-cloud-controller-manager-result.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-ns.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/local-path-storage-result.yaml - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/_deployments/vars/coredns-local.yaml diff --git a/hetzner/variables.tf b/hetzner/variables.tf index d99a61f..fbd05f7 100644 --- a/hetzner/variables.tf +++ b/hetzner/variables.tf @@ -25,19 +25,18 @@ variable "regions" { default = ["nbg1", "fsn1", "hel1"] } -variable "kubernetes" { - type = map(string) - default = { - podSubnets = "10.32.0.0/12,fd40:10:32::/102" - serviceSubnets = "10.200.0.0/22,fd40:10:200::/112" - apiDomain = "api.cluster.local" - domain = "cluster.local" - clusterName = "talos-k8s-hetzner" - tokenMachine = "" - caMachine = "" - token = "" - ca = "" - } +variable "arch" { + description = "The Talos architecture list" + type = list(string) + default = ["amd64", "arm64"] +} + +data "sops_file" "tfvars" { + source_file = "terraform.tfvars.sops.json" +} + +locals { + kubernetes = jsondecode(data.sops_file.tfvars.raw)["kubernetes"] } variable "vpc_main_cidr" { @@ -65,7 +64,6 @@ variable "controlplane" { "all" = { type_lb = "" # lb11, if "" use floating-ip }, - "nbg1" = { count = 0, type = "cpx11", @@ -85,6 +83,9 @@ variable "instances" { description = "Map of region's properties" type = map(any) default = { + "all" = { + version = "v1.30.2" + }, "nbg1" = { web_count = 0, web_type = "cx11", diff --git a/hetzner/versions.tf b/hetzner/versions.tf index 4c5e592..595c224 100644 --- a/hetzner/versions.tf +++ b/hetzner/versions.tf @@ -2,8 +2,12 @@ terraform { required_providers { hcloud = { source = "hetznercloud/hcloud" - version = "~> 1.38.2" + version = "~> 1.45" + } + sops = { + source = "carlpett/sops" + version = "1.0.0" } } - required_version = ">= 1.2" + required_version = ">= 1.5" } diff --git a/proxmox/variables.tf b/proxmox/variables.tf index 34c1ba1..e496432 100644 --- a/proxmox/variables.tf +++ b/proxmox/variables.tf @@ -38,7 +38,7 @@ variable "vpc_main_cidr" { variable "release" { type = string description = "The version of the Talos image" - default = "1.7.4" + default = "1.8.0" } data "sops_file" "tfvars" { @@ -97,7 +97,7 @@ variable "instances" { type = map(any) default = { "all" = { - version = "v1.30.2" + version = "v1.31.0" }, "hvm-1" = { enabled = false, diff --git a/scaleway/images/variables.pkr.hcl b/scaleway/images/variables.pkr.hcl index 49b71c7..b6d84e5 100644 --- a/scaleway/images/variables.pkr.hcl +++ b/scaleway/images/variables.pkr.hcl @@ -29,7 +29,7 @@ variable "scaleway_type" { variable "talos_version" { type = string - default = "v1.7.6" + default = "v1.8.0" } locals {