add worker nodes

scaleway/.gitignore

@@ -1,3 +1,10 @@
 _cfgs/
-templates/controlplane.yaml
+.terraform.lock.hcl
-*.patch
+.terraform.tfstate.lock.info
+/terraform.tfstate
+terraform.tfstate.backup
+terraform.tfvars
+terraform.tfvars.json
+terraform.tfvars.sops.json
+#
+age.key.txt

scaleway/deployments/talos-ccm.yaml

@@ -1,7 +1,7 @@
 
-# image:
+image:
   # repository: ghcr.io/sergelogvinov/talos-cloud-controller-manager
-#   tag: nodeipam
+  tag: edge
 
 service:
   containerPort: 50258

scaleway/instances-controlplane.tf

@@ -57,10 +57,11 @@ resource "scaleway_instance_server" "controlplane" {
 resource "local_sensitive_file" "controlplane" {
   count = lookup(var.controlplane, "count", 0)
   content = templatefile("${path.module}/templates/controlplane.yaml.tpl",
-    merge(var.kubernetes, try(var.instances["all"], {}), {
+    merge(local.kubernetes, try(var.instances["all"], {}), {
       name = "controlplane-${count.index + 1}"
       # labels      = local.controlplane_labels
-      nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet]
+      # nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet]
+      nodeSubnets = ["${split("/", scaleway_ipam_ip.controlplane_v4[count.index].address)[0]}/32", one(scaleway_vpc_private_network.main.ipv6_subnets).subnet]
       ipv4_local  = scaleway_ipam_ip.controlplane_v4[count.index].address
       ipv4_vip    = local.ipv4_vip
 
@@ -74,6 +75,8 @@ resource "local_sensitive_file" "controlplane" {
   )
   filename        = "_cfgs/controlplane-${count.index + 1}.yaml"
   file_permission = "0600"
+
+  depends_on = [scaleway_instance_server.controlplane]
 }
 
 locals {

scaleway/instances-web.tf

@@ -1,33 +1,46 @@
 
 locals {
-  web_labels = "topology.kubernetes.io/region=fr-par,topology.kubernetes.io/zone=${var.regions[0]},project.io/node-pool=web"
+  web_prefix = "web"
+  web_labels = "node-pool=web"
+}
+
+resource "scaleway_instance_placement_group" "web" {
+  name        = "web"
+  policy_type = "max_availability"
+  policy_mode = "enforced"
+}
+
+resource "scaleway_instance_ip" "web_v6" {
+  count = lookup(try(var.instances[var.regions[0]], {}), "web_count", 0)
+  type  = "routed_ipv6"
 }
 
 resource "scaleway_instance_server" "web" {
-  count              = lookup(var.instances, "web_count", 0)
+  count              = lookup(try(var.instances[var.regions[0]], {}), "web_count", 0)
-  name               = "web-${count.index + 1}"
+  name               = "${local.web_prefix}-${count.index + 1}"
-  image              = data.scaleway_instance_image.talos.id
+  image              = data.scaleway_instance_image.talos[length(regexall("^COPARM1", lookup(try(var.instances[var.regions[0]], {}), "web_type", 0))) > 0 ? "arm64" : "amd64"].id
-  type               = lookup(var.instances, "web_type", "DEV1-M")
+  type               = lookup(var.instances[var.regions[0]], "web_type", "DEV1-M")
-  enable_ipv6        = true
-  enable_dynamic_ip  = false
   security_group_id  = scaleway_instance_security_group.web.id
   placement_group_id = scaleway_instance_placement_group.web.id
   tags               = concat(var.tags, ["web"])
 
+  routed_ip_enabled = true
+  ip_ids            = [scaleway_instance_ip.web_v6[count.index].id]
+
   private_network {
     pn_id = scaleway_vpc_private_network.main.id
   }
 
+  root_volume {
+    size_in_gb = 20
+  }
+
   user_data = {
     cloud-init = templatefile("${path.module}/templates/worker.yaml.tpl",
-      merge(var.kubernetes, {
+      merge(local.kubernetes, try(var.instances["all"], {}), {
-        name        = "web-${count.index + 1}"
         ipv4_vip    = local.ipv4_vip
-        ipv4        = cidrhost(local.main_subnet, 21 + count.index)
+        nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet]
-        ipv4_gw     = cidrhost(local.main_subnet, 1)
+        labels      = local.web_labels
-        clusterDns  = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10)
-        nodeSubnets = local.main_subnet
-        labels      = "${local.web_labels},node.kubernetes.io/instance-type=${lookup(var.instances, "web_type", "DEV1-M")}"
       })
     )
   }
@@ -40,16 +53,3 @@ resource "scaleway_instance_server" "web" {
     ]
   }
 }
-
-resource "scaleway_instance_placement_group" "web" {
-  name        = "web"
-  policy_type = "max_availability"
-  policy_mode = "enforced"
-}
-
-resource "scaleway_vpc_public_gateway_dhcp_reservation" "web" {
-  count              = lookup(var.instances, "web_count", 0)
-  gateway_network_id = scaleway_vpc_gateway_network.main.id
-  mac_address        = scaleway_instance_server.web[count.index].private_network.0.mac_address
-  ip_address         = cidrhost(local.main_subnet, 21 + count.index)
-}

scaleway/instances-worker.tf

@@ -1,32 +1,39 @@
 
 locals {
-  worker_labels = "topology.kubernetes.io/region=fr-par,topology.kubernetes.io/zone=${var.regions[0]},project.io/node-pool=worker"
+  worker_prefix = "worker"
+  worker_labels = "node-pool=worker"
+}
+
+resource "scaleway_instance_ip" "worker_v6" {
+  count = lookup(try(var.instances[var.regions[0]], {}), "worker_count", 0)
+  type  = "routed_ipv6"
 }
 
 resource "scaleway_instance_server" "worker" {
-  count             = lookup(var.instances, "worker_count", 0)
+  count             = lookup(try(var.instances[var.regions[0]], {}), "worker_count", 0)
-  name              = "worker-${count.index + 1}"
+  name              = "${local.worker_prefix}-${count.index + 1}"
-  image             = data.scaleway_instance_image.talos.id
+  image             = data.scaleway_instance_image.talos[length(regexall("^COPARM1", lookup(try(var.instances[var.regions[0]], {}), "worker_type", 0))) > 0 ? "arm64" : "amd64"].id
-  type              = lookup(var.instances, "worker_type", "DEV1-M")
+  type              = lookup(var.instances[var.regions[0]], "worker_type", "DEV1-M")
-  enable_ipv6       = true
-  enable_dynamic_ip = false
   security_group_id = scaleway_instance_security_group.worker.id
   tags              = concat(var.tags, ["worker"])
 
+  routed_ip_enabled = true
+  ip_ids            = [scaleway_instance_ip.worker_v6[count.index].id]
+
   private_network {
     pn_id = scaleway_vpc_private_network.main.id
   }
 
+  root_volume {
+    size_in_gb = 20
+  }
+
   user_data = {
     cloud-init = templatefile("${path.module}/templates/worker.yaml.tpl",
-      merge(var.kubernetes, {
+      merge(local.kubernetes, try(var.instances["all"], {}), {
-        name        = "worker-${count.index + 1}"
         ipv4_vip    = local.ipv4_vip
-        ipv4        = cidrhost(local.main_subnet, 31 + count.index)
+        nodeSubnets = [one(scaleway_vpc_private_network.main.ipv4_subnet).subnet, one(scaleway_vpc_private_network.main.ipv6_subnets).subnet]
-        ipv4_gw     = cidrhost(local.main_subnet, 1)
+        labels      = local.worker_labels
-        clusterDns  = cidrhost(split(",", var.kubernetes["serviceSubnets"])[0], 10)
-        nodeSubnets = local.main_subnet
-        labels      = "${local.worker_labels},node.kubernetes.io/instance-type=${lookup(var.instances, "worker_type", "DEV1-M")}"
       })
     )
   }
@@ -39,10 +46,3 @@ resource "scaleway_instance_server" "worker" {
     ]
   }
 }
-
-resource "scaleway_vpc_public_gateway_dhcp_reservation" "worker" {
-  count              = lookup(var.instances, "worker_count", 0)
-  gateway_network_id = scaleway_vpc_gateway_network.main.id
-  mac_address        = scaleway_instance_server.worker[count.index].private_network.0.mac_address
-  ip_address         = cidrhost(local.main_subnet, 31 + count.index)
-}

scaleway/templates/controlplane.yaml.tpl

@@ -25,6 +25,11 @@ machine:
     kubespan:
       enabled: false
       allowDownPeerBypass: true
+      filters:
+        endpoints:
+          - 0.0.0.0/0
+          - "!${ipv4_vip}/32"
+          - "!${ipv4_local}/32"
     extraHostEntries:
       - ip: 127.0.0.1
         aliases:
@@ -83,7 +88,7 @@ cluster:
     image: registry.k8s.io/kube-controller-manager:${version}
     extraArgs:
         node-cidr-mask-size-ipv4: "24"
-        node-cidr-mask-size-ipv6: "80"
+        node-cidr-mask-size-ipv6: "112"
   scheduler:
     image: registry.k8s.io/kube-scheduler:${version}
   etcd:

scaleway/templates/worker.yaml.tpl

@@ -6,40 +6,24 @@ machine:
   token: ${tokenMachine}
   ca:
     crt: ${caMachine}
-  nodeLabels:
-    node.kubernetes.io/disktype: ssd
   kubelet:
+    image: ghcr.io/siderolabs/kubelet:${version}
     extraArgs:
       cloud-provider: external
       rotate-server-certificates: true
       node-labels: ${labels}
     clusterDNS:
       - 169.254.2.53
-      - ${clusterDns}
+      - ${cidrhost(split(",",serviceSubnets)[0], 10)}
     nodeIP:
-      validSubnets: ${format("%#v",split(",",nodeSubnets))}
+      validSubnets: ${format("%#v",nodeSubnets)}
   network:
-    hostname: "${name}"
     interfaces:
-      - interface: eth0
-        dhcp: true
-        dhcpOptions:
-          routeMetric: 2048
-        routes:
-          - network: 169.254.42.42/32
-            metric: 1024
-      - interface: eth1
-        addresses:
-          - ${ipv4}/24
-        routes:
-          - network: 0.0.0.0/0
-            gateway: ${ipv4_gw}
-            metric: 512
       - interface: dummy0
         addresses:
           - 169.254.2.53/32
     kubespan:
-      enabled: true
+      enabled: false
       allowDownPeerBypass: true
     extraHostEntries:
       - ip: ${ipv4_vip}
@@ -71,7 +55,7 @@ cluster:
     endpoint: https://${apiDomain}:6443
   clusterName: ${clusterName}
   discovery:
-    enabled: true
+    enabled: false
   network:
     dnsDomain: ${domain}
     serviceSubnets: ${format("%#v",split(",",serviceSubnets))}

scaleway/variables.tf

@@ -29,22 +29,12 @@ variable "arch" {
   default     = ["amd64", "arm64"]
 }
 
-variable "kubernetes" {
+data "sops_file" "tfvars" {
-  type = map(string)
+  source_file = "terraform.tfvars.sops.json"
-  default = {
-    podSubnets     = "10.32.0.0/12,fd40:10:32::/102"
-    serviceSubnets = "10.200.0.0/22,fd40:10:200::/112"
-    domain         = "cluster.local"
-    apiDomain      = "api.cluster.local"
-    clusterName    = "talos-k8s-scaleway"
-    clusterID      = ""
-    clusterSecret  = ""
-    tokenMachine   = ""
-    caMachine      = ""
-    token          = ""
-    ca             = ""
 }
-  sensitive = true
+
+locals {
+  kubernetes = jsondecode(data.sops_file.tfvars.raw)["kubernetes"]
 }
 
 variable "vpc_main_cidr" {
@@ -57,7 +47,7 @@ variable "controlplane" {
   description = "Property of controlplane"
   type        = map(any)
   default = {
-    count   = 1,
+    count   = 0,
     type    = "COPARM1-2C-8G" # "DEV1-L",
     type_lb = ""              # "LB-S"
   }
@@ -70,10 +60,12 @@ variable "instances" {
     "all" = {
       version = "v1.30.2"
     },
-    # web_count    = 0,
+    "fr-par-2" = {
-    # web_type     = "DEV1-L",
+      web_count    = 0,
-    # worker_count = 0,
+      web_type     = "DEV1-L",
-    # worker_type  = "DEV1-L",
+      worker_count = 0,
+      worker_type  = "COPARM1-2C-8G",
+    },
   }
 }
 

scaleway/versions.tf

@@ -5,6 +5,10 @@ terraform {
       source  = "scaleway/scaleway"
       version = "~> 2.43.0"
     }
+    sops = {
+      source  = "carlpett/sops"
+      version = "1.0.0"
+    }
   }
   required_version = ">= 1.0"
 }