scottk/terraform-talos
1
0
Fork 0
mirror of https://github.com/optim-enterprises-bv/terraform-talos.git synced 2025-10-29 01:22:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

worker nodes

Browse Source
This commit is contained in:
Serge Logvinov
2024-08-14 19:09:55 +03:00
parent 358858c7c2
commit 71dbd19fdc
16 changed files with 61 additions and 372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
hetzner/deployments/hcloud-ccm.yaml
View File

@@ -0,0 +1,5 @@
args:
controllers: cloud-node-lifecycle
nodeSelector:
node-role.kubernetes.io/control-plane: ""

4
hetzner/deployments/hcloud-cloud-controller-manager-result.yaml
View File

@@ -59,11 +59,15 @@ spec:
- key: "node.kubernetes.io/not-ready"
effect: "NoExecute"
nodeSelector:
node-role.kubernetes.io/control-plane: ""
containers:
- name: hcloud-cloud-controller-manager
args:
- "--allow-untagged-cloud"
- "--cloud-provider=hcloud"
- "--controllers=cloud-node-lifecycle"
- "--route-reconciliation-period=30s"
- "--webhook-secure-port=0"
- "--leader-elect=false"

89
hetzner/deployments/hcloud-cloud-controller-manager.yaml
View File

@@ -1,89 +0,0 @@
# NOTE: this release was tested against kubernetes v1.18.x
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: kube-system
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hcloud-cloud-controller-manager
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 2
selector:
matchLabels:
app: hcloud-cloud-controller-manager
template:
metadata:
labels:
app: hcloud-cloud-controller-manager
spec:
priorityClassName: system-cluster-critical
serviceAccountName: cloud-controller-manager
dnsPolicy: Default
nodeSelector:
node-role.kubernetes.io/control-plane: ""
node.cloudprovider.kubernetes.io/platform: hcloud
tolerations:
- key: "node.cloudprovider.kubernetes.io/uninitialized"
value: "true"
effect: "NoSchedule"
- key: "node-role.kubernetes.io/control-plane"
effect: NoSchedule
containers:
- image: hetznercloud/hcloud-cloud-controller-manager:v1.17.2
name: hcloud-cloud-controller-manager
args:
- --cloud-provider=hcloud
- --allow-untagged-cloud
- --controllers=cloud-node-lifecycle
resources:
requests:
cpu: 100m
memory: 50Mi
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: HCLOUD_TOKEN
valueFrom:
secretKeyRef:
name: hcloud
key: token
- name: HCLOUD_NETWORK
valueFrom:
secretKeyRef:
name: hcloud
key: network
- name: HCLOUD_INSTANCES_ADDRESS_FAMILY
value: dualstack
- name: ROBOT_USER_NAME
valueFrom:
secretKeyRef:
optional: true
name: hcloud
key: user
- name: ROBOT_PASSWORD
valueFrom:
secretKeyRef:
optional: true
name: hcloud
key: password

3
hetzner/instances-controlplane.tf
View File

@@ -1,11 +1,12 @@
locals {
contolplane_prefix = "controlplane"
contolplane_labels = ""
controlplanes = { for k in flatten([
for regions in var.regions : [
for inx in range(lookup(try(var.controlplane[regions], {}), "count", 0)) : {
name : "controlplane-${regions}-${1 + inx}"
name : "${local.contolplane_prefix}-${regions}-${1 + inx}"
image : data.hcloud_image.talos[startswith(lookup(try(var.controlplane[regions], {}), "type", "cpx11"), "ca") ? "arm64" : "amd64"].id
region : regions
type : lookup(try(var.controlplane[regions], {}), "type", "cpx11")

7
hetzner/instances-web.tf
View File

@@ -27,7 +27,7 @@ resource "hcloud_server" "web" {
labels = merge(var.tags, { label = "web" })
user_data = templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, {
merge(local.kubernetes, try(var.instances["all"], {}), {
name = each.value.name
ipv4 = each.value.ip
lbv4 = local.ipv4_vip
@@ -41,6 +41,10 @@ resource "hcloud_server" "web" {
network_id = hcloud_network.main.id
ip = each.value.ip
}
public_net {
ipv4_enabled = true
ipv6_enabled = true
}
lifecycle {
ignore_changes = [
@@ -48,6 +52,7 @@ resource "hcloud_server" "web" {
server_type,
user_data,
ssh_keys,
public_net,
]
}
}

6
hetzner/instances-worker-as.tf
View File

@@ -1,10 +1,10 @@
resource "local_sensitive_file" "worker-as" {
content = templatefile("${path.module}/templates/worker-as.yaml.tpl",
merge(var.kubernetes, {
content = templatefile("${path.module}/templates/worker.yaml.tpl",
merge(local.kubernetes, try(var.instances["all"], {}), {
lbv4 = local.ipv4_vip
nodeSubnets = var.vpc_main_cidr
labels = "project.io/node-pool=worker,hcloud/node-group=worker-as"
labels = "${local.worker_labels},hcloud/node-group=worker-as"
})
)

29
hetzner/instances-worker.tf
View File

@@ -27,7 +27,7 @@ resource "hcloud_server" "worker" {
labels = merge(var.tags, { label = "worker" })
user_data = templatefile("${path.module}/templates/worker.yaml.tpl",
merge(var.kubernetes, {
merge(local.kubernetes, try(var.instances["all"], {}), {
name = each.value.name
ipv4 = each.value.ip
lbv4 = local.ipv4_vip
@@ -41,6 +41,10 @@ resource "hcloud_server" "worker" {
network_id = hcloud_network.main.id
ip = each.value.ip
}
public_net {
ipv4_enabled = true
ipv6_enabled = true
}
lifecycle {
ignore_changes = [
@@ -48,28 +52,7 @@ resource "hcloud_server" "worker" {
server_type,
user_data,
ssh_keys,
public_net,
]
}
}
# module "worker" {
# source = "./modules/worker"
# for_each = var.instances
# location = each.key
# labels = merge(var.tags, { label = "worker" })
# network = hcloud_network.main.id
# subnet = hcloud_network_subnet.core.ip_range
# vm_name = "worker-${each.key}-"
# vm_items = lookup(each.value, "worker_count", 0)
# vm_type = lookup(each.value, "worker_type", "cx11")
# vm_image = data.hcloud_image.talos.id
# vm_ip_start = (6 + try(index(var.regions, each.key), 0)) * 10
# vm_security_group = [hcloud_firewall.worker.id]
# vm_params = merge(var.kubernetes, {
# lbv4 = local.ipv4_vip
# labels = "project.io/node-pool=worker,hcloud/node-group=worker-${each.key}"
# })
# }

64
hetzner/modules/templates/worker-as.yaml.tpl
View File

@@ -1,64 +0,0 @@
version: v1alpha1
debug: false
persist: true
machine:
type: worker
token: ${tokenMachine}
ca:
crt: ${caMachine}
certSANs: []
nodeLabels:
node.kubernetes.io/disktype: ssd
kubelet:
extraArgs:
cloud-provider: external
rotate-server-certificates: true
node-labels: "${labels}"
clusterDNS:
- 169.254.2.53
- ${cidrhost(split(",",serviceSubnets)[0], 10)}
nodeIP:
validSubnets: ${format("%#v",split(",",nodeSubnets))}
network:
interfaces:
- interface: dummy0
addresses:
- 169.254.2.53/32
extraHostEntries:
- ip: ${lbv4}
aliases:
- ${apiDomain}
sysctls:
net.core.somaxconn: 65535
net.core.netdev_max_backlog: 4096
install:
wipe: false
systemDiskEncryption:
state:
provider: luks2
keys:
- nodeID: {}
slot: 0
ephemeral:
provider: luks2
keys:
- nodeID: {}
slot: 0
options:
- no_read_workqueue
- no_write_workqueue
cluster:
id: ${clusterID}
secret: ${clusterSecret}
controlPlane:
endpoint: https://${apiDomain}:6443
clusterName: ${clusterName}
discovery:
enabled: true
network:
dnsDomain: ${domain}
podSubnets: ${format("%#v",split(",",podSubnets))}
serviceSubnets: ${format("%#v",split(",",serviceSubnets))}
token: ${token}
ca:
crt: ${ca}

65
hetzner/modules/templates/worker.yaml.tpl
View File

@@ -1,65 +0,0 @@
version: v1alpha1
debug: false
persist: true
machine:
type: worker
token: ${tokenMachine}
ca:
crt: ${caMachine}
certSANs: []
nodeLabels:
node.kubernetes.io/disktype: ssd
kubelet:
extraArgs:
node-ip: "${ipv4}"
cloud-provider: external
rotate-server-certificates: true
node-labels: "${labels}"
clusterDNS:
- 169.254.2.53
- ${cidrhost(split(",",serviceSubnets)[0], 10)}
nodeIP:
validSubnets: ${format("%#v",split(",",nodeSubnets))}
network:
hostname: "${name}"
interfaces:
- interface: dummy0
addresses:
- 169.254.2.53/32
extraHostEntries:
- ip: ${lbv4}
aliases:
- ${apiDomain}
install:
wipe: false
sysctls:
net.core.somaxconn: 65535
net.core.netdev_max_backlog: 4096
systemDiskEncryption:
state:
provider: luks2
keys:
- nodeID: {}
slot: 0
ephemeral:
provider: luks2
keys:
- nodeID: {}
slot: 0
options:
- no_read_workqueue
- no_write_workqueue
cluster:
id: ${clusterID}
secret: ${clusterSecret}
controlPlane:
endpoint: https://${apiDomain}:6443
clusterName: ${clusterName}
discovery:
enabled: true
network:
dnsDomain: ${domain}
serviceSubnets: ${format("%#v",split(",",serviceSubnets))}
token: ${token}
ca:
crt: ${ca}

55
hetzner/modules/worker/main.tf
View File

@@ -1,55 +0,0 @@
resource "hcloud_server" "worker" {
count = var.vm_items
location = var.location
name = "${var.vm_name}${count.index + 1}"
image = var.vm_image
server_type = var.vm_type
ssh_keys = []
keep_disk = true
labels = var.labels
user_data = templatefile("${path.module}/../templates/worker.yaml.tpl",
merge(var.vm_params, {
name = "${var.vm_name}${count.index + 1}"
ipv4 = cidrhost(var.subnet, var.vm_ip_start + count.index)
})
)
firewall_ids = var.vm_security_group
network {
network_id = var.network
ip = cidrhost(var.subnet, var.vm_ip_start + count.index)
}
lifecycle {
ignore_changes = [
image,
server_type,
user_data,
ssh_keys,
]
}
# IPv6 hack
# provisioner "local-exec" {
# command = "echo '${templatefile("${path.module}/../templates/worker-patch.json.tpl", { ipv6_address = self.ipv6_address })}' > _cfgs/${var.vm_name}${count.index + 1}.patch"
# }
# provisioner "local-exec" {
# command = "sleep 120 && talosctl --talosconfig _cfgs/talosconfig patch --nodes ${cidrhost(var.subnet, var.vm_ip_start + count.index)} machineconfig --patch-file _cfgs/${var.vm_name}${count.index + 1}.patch"
# }
}
# resource "local_file" "worker" {
# count = var.vm_items
# content = templatefile("${path.module}/../templates/worker.yaml.tpl",
# merge(var.vm_params, {
# name = "${var.vm_name}${count.index + 1}"
# ipv4 = cidrhost(var.subnet, var.vm_ip_start + count.index)
# })
# )
# filename = "${var.vm_name}${count.index + 1}.yaml"
# file_permission = "0640"
# depends_on = [hcloud_server.worker]
# }

4
hetzner/modules/worker/outputs.tf
View File

@@ -1,4 +0,0 @@
output "vms" {
value = hcloud_server.worker
}

52
hetzner/modules/worker/variables.tf
View File

@@ -1,52 +0,0 @@
variable "location" {
type = string
default = "nbg1"
}
variable "labels" {
type = map(string)
description = "Tags of resources"
}
variable "network" {
type = string
description = "Network id"
}
variable "subnet" {
type = string
description = "Subnet cidr"
}
variable "vm_name" {
type = string
default = "worker-"
}
variable "vm_items" {
type = number
default = 0
}
variable "vm_type" {
type = string
default = "cx11"
}
variable "vm_image" {
type = string
}
variable "vm_security_group" {
type = list(string)
}
variable "vm_ip_start" {
type = number
default = 61
}
variable "vm_params" {
type = map(string)
}

9
hetzner/modules/worker/versions.tf
View File

@@ -1,9 +0,0 @@
terraform {
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "~> 1.36.1"
}
}
required_version = ">= 1.2"
}

6
hetzner/network-lb.tf
View File

@@ -18,6 +18,12 @@ resource "hcloud_floating_ip" "api" {
labels = merge(var.tags, { type = "infra" })
}
# resource "hcloud_floating_ip_assignment" "api" {
# count = local.lb_enable ? 0 : 1
# floating_ip_id = hcloud_floating_ip.api[0].id
# server_id = one(hcloud_server.controlplane).id
# }
resource "hcloud_load_balancer" "api" {
count = local.lb_enable ? 1 : 0
name = "api"

26
hetzner/templates/worker.yaml.tpl
View File

@@ -6,30 +6,40 @@ machine:
token: ${tokenMachine}
ca:
crt: ${caMachine}
nodeLabels:
node.kubernetes.io/disktype: ssd
kubelet:
image: ghcr.io/siderolabs/kubelet:${version}
defaultRuntimeSeccompProfileEnabled: true
extraArgs:
cloud-provider: external
rotate-server-certificates: true
node-labels: "${labels}"
node-labels: ${labels}
extraConfig:
imageGCHighThresholdPercent: 70
imageGCLowThresholdPercent: 50
allowedUnsafeSysctls: [net.core.somaxconn]
clusterDNS:
- 169.254.2.53
- ${cidrhost(split(",",serviceSubnets)[0], 10)}
nodeIP:
validSubnets: ${format("%#v",split(",",nodeSubnets))}
network:
hostname: "${name}"
interfaces:
- interface: dummy0
addresses:
- 169.254.2.53/32
kubespan:
enabled: false
allowDownPeerBypass: true
extraHostEntries:
- ip: ${lbv4}
aliases:
- ${apiDomain}
nameservers:
- 1.1.1.1
- 2606:4700:4700::1111
- 2001:4860:4860::8888
install:
wipe: false
wipe: true
sysctls:
net.core.somaxconn: 65535
net.core.netdev_max_backlog: 4096
@@ -47,6 +57,10 @@ machine:
options:
- no_read_workqueue
- no_write_workqueue
features:
rbac: true
stableHostname: true
apidCheckExtKeyUsage: true
cluster:
id: ${clusterID}
secret: ${clusterSecret}
@@ -54,7 +68,7 @@ cluster:
endpoint: https://${apiDomain}:6443
clusterName: ${clusterName}
discovery:
enabled: true
enabled: false
network:
dnsDomain: ${domain}
serviceSubnets: ${format("%#v",split(",",serviceSubnets))}

9
scaleway/templates/worker.yaml.tpl
View File

@@ -8,10 +8,15 @@ machine:
crt: ${caMachine}
kubelet:
image: ghcr.io/siderolabs/kubelet:${version}
defaultRuntimeSeccompProfileEnabled: true
extraArgs:
cloud-provider: external
rotate-server-certificates: true
node-labels: ${labels}
extraConfig:
imageGCHighThresholdPercent: 70
imageGCLowThresholdPercent: 50
allowedUnsafeSysctls: [net.core.somaxconn]
clusterDNS:
- 169.254.2.53
- ${cidrhost(split(",",serviceSubnets)[0], 10)}
@@ -48,6 +53,10 @@ machine:
options:
- no_read_workqueue
- no_write_workqueue
features:
rbac: true
stableHostname: true
apidCheckExtKeyUsage: true
cluster:
id: ${clusterID}
secret: ${clusterSecret}