diff --git a/scaleway/.gitignore b/scaleway/.gitignore index 01eee69..0872560 100644 --- a/scaleway/.gitignore +++ b/scaleway/.gitignore @@ -1,3 +1,10 @@ _cfgs/ -templates/controlplane.yaml -*.patch +.terraform.lock.hcl +.terraform.tfstate.lock.info +/terraform.tfstate +terraform.tfstate.backup +terraform.tfvars +terraform.tfvars.json +terraform.tfvars.sops.json +# +age.key.txt diff --git a/scaleway/deployments/talos-ccm.yaml b/scaleway/deployments/talos-ccm.yaml index 1fa3a5b..aed679f 100644 --- a/scaleway/deployments/talos-ccm.yaml +++ b/scaleway/deployments/talos-ccm.yaml @@ -1,7 +1,7 @@ -# image: -# repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager -# tag: nodeipam +image: + # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager + tag: edge service: containerPort: 50258 diff --git a/scaleway/instances-controlplane.tf b/scaleway/instances-controlplane.tf index 3862425..63c3db4 100644 --- a/scaleway/instances-controlplane.tf +++ b/scaleway/instances-controlplane.tf @@ -57,10 +57,11 @@ resource "scaleway_instance_server" "controlplane" { resource "local_sensitive_file" "controlplane" { count = lookup(var.controlplane, "count", 0) content = templatefile("${path.module}/templates/controlplane.yaml.tpl", - merge(var.kubernetes, try(var.instances["all"], {}), { + merge(local.kubernetes, try(var.instances["all"], {}), { name = "controlplane-${count.index + 1}" # labels = local.controlplane_labels - nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet] + # nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet] + nodeSubnets = ["${split("/", scaleway_ipam_ip.controlplane_v4[count.index].address)[0]}/32", one(scaleway_vpc_private_network.main.ipv6_subnets).subnet] ipv4_local = scaleway_ipam_ip.controlplane_v4[count.index].address ipv4_vip = local.ipv4_vip @@ -74,6 +75,8 @@ resource "local_sensitive_file" "controlplane" { ) filename = "_cfgs/controlplane-${count.index + 1}.yaml" file_permission = "0600" + + depends_on = [scaleway_instance_server.controlplane] } locals { diff --git a/scaleway/instances-web.tf b/scaleway/instances-web.tf index 9fe5a4e..a466671 100644 --- a/scaleway/instances-web.tf +++ b/scaleway/instances-web.tf @@ -1,33 +1,46 @@ locals { - web_labels = "topology.kubernetes.io/region=fr-par,topology.kubernetes.io/zone=${var.regions[0]},project.io/node-pool=web" + web_prefix = "web" + web_labels = "node-pool=web" +} + +resource "scaleway_instance_placement_group" "web" { + name = "web" + policy_type = "max_availability" + policy_mode = "enforced" +} + +resource "scaleway_instance_ip" "web_v6" { + count = lookup(try(var.instances[var.regions[0]], {}), "web_count", 0) + type = "routed_ipv6" } resource "scaleway_instance_server" "web" { - count = lookup(var.instances, "web_count", 0) - name = "web-${count.index + 1}" - image = data.scaleway_instance_image.talos.id - type = lookup(var.instances, "web_type", "DEV1-M") - enable_ipv6 = true - enable_dynamic_ip = false + count = lookup(try(var.instances[var.regions[0]], {}), "web_count", 0) + name = "${local.web_prefix}-${count.index + 1}" + image = data.scaleway_instance_image.talos[length(regexall("^COPARM1", lookup(try(var.instances[var.regions[0]], {}), "web_type", 0))) > 0 ? "arm64" : "amd64"].id + type = lookup(var.instances[var.regions[0]], "web_type", "DEV1-M") security_group_id = scaleway_instance_security_group.web.id placement_group_id = scaleway_instance_placement_group.web.id tags = concat(var.tags, ["web"]) + routed_ip_enabled = true + ip_ids = [scaleway_instance_ip.web_v6[count.index].id] + private_network { pn_id = scaleway_vpc_private_network.main.id } + root_volume { + size_in_gb = 20 + } + user_data = { cloud-init = templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, { - name = "web-${count.index + 1}" + merge(local.kubernetes, try(var.instances["all"], {}), { ipv4_vip = local.ipv4_vip - ipv4 = cidrhost(local.main_subnet, 21 + count.index) - ipv4_gw = cidrhost(local.main_subnet, 1) - clusterDns = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10) - nodeSubnets = local.main_subnet - labels = "${local.web_labels},node.kubernetes.io/instance-type=${lookup(var.instances, "web_type", "DEV1-M")}" + nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet] + labels = local.web_labels }) ) } @@ -40,16 +53,3 @@ resource "scaleway_instance_server" "web" { ] } } - -resource "scaleway_instance_placement_group" "web" { - name = "web" - policy_type = "max_availability" - policy_mode = "enforced" -} - -resource "scaleway_vpc_public_gateway_dhcp_reservation" "web" { - count = lookup(var.instances, "web_count", 0) - gateway_network_id = scaleway_vpc_gateway_network.main.id - mac_address = scaleway_instance_server.web[count.index].private_network.0.mac_address - ip_address = cidrhost(local.main_subnet, 21 + count.index) -} diff --git a/scaleway/instances-worker.tf b/scaleway/instances-worker.tf index 0ab73b3..3e7af95 100644 --- a/scaleway/instances-worker.tf +++ b/scaleway/instances-worker.tf @@ -1,32 +1,39 @@ locals { - worker_labels = "topology.kubernetes.io/region=fr-par,topology.kubernetes.io/zone=${var.regions[0]},project.io/node-pool=worker" + worker_prefix = "worker" + worker_labels = "node-pool=worker" +} + +resource "scaleway_instance_ip" "worker_v6" { + count = lookup(try(var.instances[var.regions[0]], {}), "worker_count", 0) + type = "routed_ipv6" } resource "scaleway_instance_server" "worker" { - count = lookup(var.instances, "worker_count", 0) - name = "worker-${count.index + 1}" - image = data.scaleway_instance_image.talos.id - type = lookup(var.instances, "worker_type", "DEV1-M") - enable_ipv6 = true - enable_dynamic_ip = false + count = lookup(try(var.instances[var.regions[0]], {}), "worker_count", 0) + name = "${local.worker_prefix}-${count.index + 1}" + image = data.scaleway_instance_image.talos[length(regexall("^COPARM1", lookup(try(var.instances[var.regions[0]], {}), "worker_type", 0))) > 0 ? "arm64" : "amd64"].id + type = lookup(var.instances[var.regions[0]], "worker_type", "DEV1-M") security_group_id = scaleway_instance_security_group.worker.id tags = concat(var.tags, ["worker"]) + routed_ip_enabled = true + ip_ids = [scaleway_instance_ip.worker_v6[count.index].id] + private_network { pn_id = scaleway_vpc_private_network.main.id } + root_volume { + size_in_gb = 20 + } + user_data = { cloud-init = templatefile("${path.module}/templates/worker.yaml.tpl", - merge(var.kubernetes, { - name = "worker-${count.index + 1}" + merge(local.kubernetes, try(var.instances["all"], {}), { ipv4_vip = local.ipv4_vip - ipv4 = cidrhost(local.main_subnet, 31 + count.index) - ipv4_gw = cidrhost(local.main_subnet, 1) - clusterDns = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10) - nodeSubnets = local.main_subnet - labels = "${local.worker_labels},node.kubernetes.io/instance-type=${lookup(var.instances, "worker_type", "DEV1-M")}" + nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet] + labels = local.worker_labels }) ) } @@ -39,10 +46,3 @@ resource "scaleway_instance_server" "worker" { ] } } - -resource "scaleway_vpc_public_gateway_dhcp_reservation" "worker" { - count = lookup(var.instances, "worker_count", 0) - gateway_network_id = scaleway_vpc_gateway_network.main.id - mac_address = scaleway_instance_server.worker[count.index].private_network.0.mac_address - ip_address = cidrhost(local.main_subnet, 31 + count.index) -} diff --git a/scaleway/templates/controlplane.yaml.tpl b/scaleway/templates/controlplane.yaml.tpl index e9df035..19513cb 100644 --- a/scaleway/templates/controlplane.yaml.tpl +++ b/scaleway/templates/controlplane.yaml.tpl @@ -25,6 +25,11 @@ machine: kubespan: enabled: false allowDownPeerBypass: true + filters: + endpoints: + - 0.0.0.0/0 + - "!${ipv4_vip}/32" + - "!${ipv4_local}/32" extraHostEntries: - ip: 127.0.0.1 aliases: @@ -83,7 +88,7 @@ cluster: image: registry.k8s.io/kube-controller-manager:${version} extraArgs: node-cidr-mask-size-ipv4: "24" - node-cidr-mask-size-ipv6: "80" + node-cidr-mask-size-ipv6: "112" scheduler: image: registry.k8s.io/kube-scheduler:${version} etcd: diff --git a/scaleway/templates/worker.yaml.tpl b/scaleway/templates/worker.yaml.tpl index 3674fe3..76b80dc 100644 --- a/scaleway/templates/worker.yaml.tpl +++ b/scaleway/templates/worker.yaml.tpl @@ -6,40 +6,24 @@ machine: token: ${tokenMachine} ca: crt: ${caMachine} - nodeLabels: - node.kubernetes.io/disktype: ssd kubelet: + image: ghcr.io/siderolabs/kubelet:${version} extraArgs: cloud-provider: external rotate-server-certificates: true node-labels: ${labels} clusterDNS: - 169.254.2.53 - - ${clusterDns} + - ${cidrhost(split(",",serviceSubnets)[0], 10)} nodeIP: - validSubnets: ${format("%#v",split(",",nodeSubnets))} + validSubnets: ${format("%#v",nodeSubnets)} network: - hostname: "${name}" interfaces: - - interface: eth0 - dhcp: true - dhcpOptions: - routeMetric: 2048 - routes: - - network: 169.254.42.42/32 - metric: 1024 - - interface: eth1 - addresses: - - ${ipv4}/24 - routes: - - network: 0.0.0.0/0 - gateway: ${ipv4_gw} - metric: 512 - interface: dummy0 addresses: - 169.254.2.53/32 kubespan: - enabled: true + enabled: false allowDownPeerBypass: true extraHostEntries: - ip: ${ipv4_vip} @@ -71,7 +55,7 @@ cluster: endpoint: https://${apiDomain}:6443 clusterName: ${clusterName} discovery: - enabled: true + enabled: false network: dnsDomain: ${domain} serviceSubnets: ${format("%#v",split(",",serviceSubnets))} diff --git a/scaleway/variables.tf b/scaleway/variables.tf index 8e3d5d3..c078e35 100644 --- a/scaleway/variables.tf +++ b/scaleway/variables.tf @@ -29,22 +29,12 @@ variable "arch" { default = ["amd64", "arm64"] } -variable "kubernetes" { - type = map(string) - default = { - podSubnets = "10.32.0.0/12,fd40:10:32::/102" - serviceSubnets = "10.200.0.0/22,fd40:10:200::/112" - domain = "cluster.local" - apiDomain = "api.cluster.local" - clusterName = "talos-k8s-scaleway" - clusterID = "" - clusterSecret = "" - tokenMachine = "" - caMachine = "" - token = "" - ca = "" - } - sensitive = true +data "sops_file" "tfvars" { + source_file = "terraform.tfvars.sops.json" +} + +locals { + kubernetes = jsondecode(data.sops_file.tfvars.raw)["kubernetes"] } variable "vpc_main_cidr" { @@ -57,7 +47,7 @@ variable "controlplane" { description = "Property of controlplane" type = map(any) default = { - count = 1, + count = 0, type = "COPARM1-2C-8G" # "DEV1-L", type_lb = "" # "LB-S" } @@ -70,10 +60,12 @@ variable "instances" { "all" = { version = "v1.30.2" }, - # web_count = 0, - # web_type = "DEV1-L", - # worker_count = 0, - # worker_type = "DEV1-L", + "fr-par-2" = { + web_count = 0, + web_type = "DEV1-L", + worker_count = 0, + worker_type = "COPARM1-2C-8G", + }, } } diff --git a/scaleway/versions.tf b/scaleway/versions.tf index 670e23d..d335795 100644 --- a/scaleway/versions.tf +++ b/scaleway/versions.tf @@ -5,6 +5,10 @@ terraform { source = "scaleway/scaleway" version = "~> 2.43.0" } + sops = { + source = "carlpett/sops" + version = "1.0.0" + } } required_version = ">= 1.0" }