Update scaleway example

.gitignore

@@ -32,3 +32,4 @@ override.tf.json
 #
 talosctl
 talosconfig
+kubeconfig

scaleway/Makefile

@@ -1,22 +1,37 @@
 
-create-config:
+ENDPOINT:=${shell terraform output -raw controlplane_endpoint 2>/dev/null}
-	./talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false talos-k8s-scaleway https://127.0.0.1:6443
+
+help:
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {sub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+create-lb: ## Create load balancer
+	terraform init
+	terraform apply -auto-approve -target=scaleway_vpc_public_gateway_ip.main -target=output.controlplane_endpoint
+
+create-config: ## Genereate talos configs
+	talosctl gen config --output-dir _cfgs --with-docs=false --with-examples=false talos-k8s-scaleway https://${ENDPOINT}:6443
+	talosctl --talosconfig _cfgs/talosconfig config endpoint ${ENDPOINT}
 
 create-templates:
 	@yq ea -P '. as $$item ireduce ({}; . * $$item )' _cfgs/controlplane.yaml templates/controlplane.yaml.tpl > templates/controlplane.yaml
 	@echo 'podSubnets: "10.32.0.0/12,fd00:10:32::/102"'        >  _cfgs/tfstate.vars
 	@echo 'serviceSubnets: "10.200.0.0/22,fd40:10:200::/112"'  >> _cfgs/tfstate.vars
+	@echo 'nodeSubnets: "172.16.0.0/12"'                       >> _cfgs/tfstate.vars
+	@echo 'apiDomain: api.cluster.local'                       >> _cfgs/tfstate.vars
 	@yq eval '.cluster.network.dnsDomain' _cfgs/controlplane.yaml | awk '{ print "domain: "$$1}'       >> _cfgs/tfstate.vars
-	@yq eval '.cluster.clusterName' _cfgs/controlplane.yaml       | awk '{ print "cluster_name: "$$1}' >> _cfgs/tfstate.vars
+	@yq eval '.cluster.clusterName' _cfgs/controlplane.yaml       | awk '{ print "clusterName: "$$1}'  >> _cfgs/tfstate.vars
-	@yq eval '.machine.token'  _cfgs/controlplane.yaml            | awk '{ print "tokenmachine: "$$1}' >> _cfgs/tfstate.vars
+	@yq eval '.machine.token'  _cfgs/controlplane.yaml            | awk '{ print "tokenMachine: "$$1}' >> _cfgs/tfstate.vars
+	@yq eval '.machine.ca.crt' _cfgs/controlplane.yaml            | awk '{ print "caMachine: "$$1}'    >> _cfgs/tfstate.vars
 	@yq eval '.cluster.token'  _cfgs/controlplane.yaml            | awk '{ print "token: "$$1}'        >> _cfgs/tfstate.vars
 	@yq eval '.cluster.ca.crt' _cfgs/controlplane.yaml            | awk '{ print "ca: "$$1}'           >> _cfgs/tfstate.vars
 
-	@yq eval -j '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json
+	@yq eval -o=json '{"kubernetes": .}' _cfgs/tfstate.vars > terraform.tfvars.json
 
-create-controlplane:
+create-controlplane: ## Bootstrap first controlplane node
 	terraform apply -target=scaleway_instance_private_nic.controlplane
 
-create-infrastructure:
+create-infrastructure: ## Bootstrap all nodes
-	cd modules/worker && terraform init
 	terraform apply
+
+create-kubeconfig: ## Prepare kubeconfig
+	talosctl --talosconfig _cfgs/talosconfig --nodes 172.16.0.11 kubeconfig .

scaleway/instances-master.tf

@@ -1,12 +1,10 @@
 
 resource "scaleway_instance_ip" "controlplane" {
   count = lookup(var.controlplane, "count", 0)
-  # zone    = element(var.regions, count.index)
 }
 
 resource "scaleway_instance_server" "controlplane" {
-  count = lookup(var.controlplane, "count", 0)
+  count             = lookup(var.controlplane, "count", 0)
-  # zone    = element(var.regions, count.index)
   name              = "master-${count.index + 1}"
   image             = data.scaleway_instance_image.talos.id
   type              = lookup(var.controlplane, "type", "DEV1-M")
@@ -15,13 +13,20 @@ resource "scaleway_instance_server" "controlplane" {
   security_group_id = scaleway_instance_security_group.controlplane.id
   tags              = concat(var.tags, ["infra", "master"])
 
+  private_network {
+    pn_id = scaleway_vpc_private_network.main.id
+  }
+
   user_data = {
     cloud-init = templatefile("${path.module}/templates/controlplane.yaml",
       merge(var.kubernetes, {
-        name = "master-${count.index + 1}"
+        name       = "master-${count.index + 1}"
-        type = count.index == 0 ? "init" : "controlplane"
+        type       = "controlplane"
-        lbv4 = local.lbv4
+        ipv4_vip   = local.ipv4_vip
-        ipv4 = scaleway_instance_ip.controlplane[count.index].address
+        ipv4_local = cidrhost(local.main_subnet, 11 + count.index)
+        lbv4       = local.lbv4
+        ipv4       = scaleway_instance_ip.controlplane[count.index].address
+        labels     = "topology.kubernetes.io/region=fr-par"
       })
     )
   }
@@ -34,9 +39,3 @@ resource "scaleway_instance_server" "controlplane" {
     ]
   }
 }
-
-resource "scaleway_instance_private_nic" "controlplane" {
-  count              = lookup(var.controlplane, "count", 0)
-  server_id          = scaleway_instance_server.controlplane[count.index].id
-  private_network_id = scaleway_vpc_private_network.main.id
-}

scaleway/network-lb.tf

@@ -1,50 +1,49 @@
 
 locals {
   lb_enable = lookup(var.controlplane, "type_lb", "") == "" ? false : true
-}
 
-locals {
+  ipv4_vip = cidrhost(local.main_subnet, 5)
-  lbv4 = local.lb_enable ? scaleway_lb_ip.lb[0].ip_address : try(scaleway_instance_ip.controlplane[0].address, "127.0.0.1")
+  lbv4     = local.lb_enable ? scaleway_lb_ip.lb[0].ip_address : scaleway_vpc_public_gateway_ip.main.address
 }
 
 resource "scaleway_lb_ip" "lb" {
   count = local.lb_enable ? 1 : 0
-  # zone    = element(var.regions, count.index)
 }
 
 resource "scaleway_lb" "lb" {
   count = local.lb_enable ? 1 : 0
-  # name  = "lb"
+  name  = "lb"
   ip_id = scaleway_lb_ip.lb[0].id
-  type  = lookup(var.controlplane, "type_lb", "")
+  type  = lookup(var.controlplane, "type_lb", "LB-S")
-  tags  = concat(var.tags, ["infra"])
+
+  tags = concat(var.tags, ["infra"])
 }
 
-resource "scaleway_lb_backend" "api" {
+# resource "scaleway_lb_backend" "api" {
-  count            = local.lb_enable ? 1 : 0
+#   count            = local.lb_enable ? 1 : 0
-  lb_id            = scaleway_lb.lb[0].id
+#   lb_id            = scaleway_lb.lb[0].id
-  name             = "api"
+#   name             = "api"
-  forward_protocol = "tcp"
+#   forward_protocol = "tcp"
-  forward_port     = "6443"
+#   forward_port     = "6443"
-  server_ips       = scaleway_instance_server.controlplane[*].private_ip
+#   server_ips       = scaleway_instance_server.controlplane[*].private_ip
 
-  health_check_tcp {}
+#   health_check_tcp {}
-}
+# }
 
-resource "scaleway_lb_frontend" "api" {
+# resource "scaleway_lb_frontend" "api" {
-  count        = local.lb_enable ? 1 : 0
+#   count        = local.lb_enable ? 1 : 0
-  lb_id        = scaleway_lb.lb[0].id
+#   lb_id        = scaleway_lb.lb[0].id
-  backend_id   = scaleway_lb_backend.api[0].id
+#   backend_id   = scaleway_lb_backend.api[0].id
-  name         = "api"
+#   name         = "api"
-  inbound_port = "6443"
+#   inbound_port = "6443"
 
-  acl {
+#   acl {
-    name = "Allow whitlist IPs"
+#     name = "Allow whitlist IPs"
-    action {
+#     action {
-      type = "allow"
+#       type = "allow"
-    }
+#     }
-    match {
+#     match {
-      ip_subnet = var.whitelist_admins
+#       ip_subnet = var.whitelist_admins
-    }
+#     }
-  }
+#   }
-}
+# }

scaleway/network-secgroup.tf

@@ -37,34 +37,7 @@ resource "scaleway_instance_security_group" "controlplane" {
 
   inbound_rule {
     action   = "accept"
-    protocol = "ICMP"
+    protocol = "UDP"
-  }
-}
-
-resource "scaleway_instance_security_group" "web" {
-  name                    = "web"
-  inbound_default_policy  = "drop"
-  outbound_default_policy = "accept"
-
-  dynamic "inbound_rule" {
-    for_each = ["80", "443"]
-
-    content {
-      action   = "accept"
-      protocol = "TCP"
-      port     = inbound_rule.value
-    }
-  }
-
-  dynamic "inbound_rule" {
-    for_each = ["4240"]
-
-    content {
-      action   = "accept"
-      protocol = "TCP"
-      port     = inbound_rule.value
-      ip_range = "::/0"
-    }
   }
 
   inbound_rule {
@@ -73,24 +46,56 @@ resource "scaleway_instance_security_group" "web" {
   }
 }
 
-resource "scaleway_instance_security_group" "worker" {
+# resource "scaleway_instance_security_group" "web" {
-  name                    = "worker"
+#   name                    = "web"
-  inbound_default_policy  = "drop"
+#   inbound_default_policy  = "drop"
-  outbound_default_policy = "accept"
+#   outbound_default_policy = "accept"
 
-  dynamic "inbound_rule" {
+#   dynamic "inbound_rule" {
-    for_each = ["4240"]
+#     for_each = ["80", "443"]
 
-    content {
+#     content {
-      action   = "accept"
+#       action   = "accept"
-      protocol = "TCP"
+#       protocol = "TCP"
-      port     = inbound_rule.value
+#       port     = inbound_rule.value
-      ip_range = "::/0"
+#     }
-    }
+#   }
-  }
 
-  inbound_rule {
+#   dynamic "inbound_rule" {
-    action   = "accept"
+#     for_each = ["4240"]
-    protocol = "ICMP"
+
-  }
+#     content {
-}
+#       action   = "accept"
+#       protocol = "TCP"
+#       port     = inbound_rule.value
+#       ip_range = "::/0"
+#     }
+#   }
+
+#   inbound_rule {
+#     action   = "accept"
+#     protocol = "ICMP"
+#   }
+# }
+
+# resource "scaleway_instance_security_group" "worker" {
+#   name                    = "worker"
+#   inbound_default_policy  = "drop"
+#   outbound_default_policy = "accept"
+
+#   dynamic "inbound_rule" {
+#     for_each = ["4240"]
+
+#     content {
+#       action   = "accept"
+#       protocol = "TCP"
+#       port     = inbound_rule.value
+#       ip_range = "::/0"
+#     }
+#   }
+
+#   inbound_rule {
+#     action   = "accept"
+#     protocol = "ICMP"
+#   }
+# }

scaleway/network.tf

@@ -1,5 +1,44 @@
 
+locals {
+  main_subnet = cidrsubnet(var.vpc_main_cidr, 8, 0)
+}
+
+resource "scaleway_vpc_public_gateway_ip" "main" {
+  tags = concat(var.tags, ["infra"])
+}
+
+resource "scaleway_vpc_public_gateway" "main" {
+  name  = "main"
+  type  = "VPC-GW-S"
+  ip_id = scaleway_vpc_public_gateway_ip.main.id
+
+  tags = concat(var.tags, ["infra"])
+}
+
+resource "scaleway_vpc_public_gateway_dhcp" "main" {
+  subnet   = local.main_subnet
+  pool_low = cidrhost(local.main_subnet, 16)
+}
+
 resource "scaleway_vpc_private_network" "main" {
   name = "main"
+
   tags = concat(var.tags, ["infra"])
 }
+
+resource "scaleway_vpc_gateway_network" "main" {
+  gateway_id         = scaleway_vpc_public_gateway.main.id
+  private_network_id = scaleway_vpc_private_network.main.id
+  dhcp_id            = scaleway_vpc_public_gateway_dhcp.main.id
+  cleanup_dhcp       = true
+}
+
+# resource "scaleway_vpc_public_gateway_pat_rule" "main" {
+#   count        = lookup(var.controlplane, "count", 0)
+#   gateway_id   = scaleway_vpc_public_gateway.main.id
+#   private_ip   = cidrhost(local.main_subnet, 11)
+#   private_port = 50000
+#   public_port  = 50000
+#   protocol     = "tcp"
+#   depends_on   = [scaleway_vpc_gateway_network.main, scaleway_vpc_private_network.main]
+# }

scaleway/outputs.tf

@@ -0,0 +1,25 @@
+
+output "controlplane_endpoint" {
+  description = "Kubernetes controlplane endpoint"
+  value       = local.lbv4
+}
+
+output "controlplane_firstnode" {
+  description = "Kubernetes controlplane first node"
+  value       = try(scaleway_instance_ip.controlplane[0].address, "none")
+}
+
+# output "controlplane_nodes" {
+#   description = "Kubernetes controlplane nodes"
+#   value = [
+#     for s in hcloud_server.controlplane[*] :
+#     {
+#       name         = s.name
+#       ipv4_address = s.ipv4_address
+#       ipv6_address = s.ipv6_address
+#       zone         = "hetzner"
+#       location     = s.location
+#       params       = ""
+#     }
+#   ]
+# }

scaleway/templates/controlplane.yaml.tpl

@@ -6,16 +6,26 @@ machine:
   certSANs:
     - "${lbv4}"
     - "${ipv4}"
+    - "${ipv4_local}"
+    - "${ipv4_vip}"
   kubelet:
     extraArgs:
+      node-ip: "${ipv4_local}"
       rotate-server-certificates: true
+      node-labels: "${labels}"
+    clusterDNS:
+      - 169.254.2.53
+      - ${cidrhost(split(",",serviceSubnets)[0], 10)}
+    nodeIP:
+      validSubnets: ${format("%#v",split(",",nodeSubnets))}
   network:
     hostname: "${name}"
     interfaces:
       - interface: eth1
-        dhcp: true
+        addresses:
-        dhcpOptions:
+          - ${ipv4_local}/24
-          routeMetric: 2048
+        vip:
+          ip: ${ipv4_vip}
       - interface: dummy0
         addresses:
           - 169.254.2.53/32
@@ -26,18 +36,26 @@ machine:
     net.core.somaxconn: 65535
     net.core.netdev_max_backlog: 4096
   systemDiskEncryption:
+    state:
+      provider: luks2
+      keys:
+        - nodeID: {}
+          slot: 0
     ephemeral:
       provider: luks2
       keys:
         - nodeID: {}
           slot: 0
+      options:
+        - no_read_workqueue
+        - no_write_workqueue
 cluster:
   controlPlane:
-    endpoint: https://${lbv4}:6443
+    endpoint: https://${ipv4_vip}:6443
   network:
     dnsDomain: ${domain}
-    podSubnets: ${format("[%s]",podSubnets)}
+    podSubnets: ${format("%#v",split(",",podSubnets))}
-    serviceSubnets: ${format("[%s]",serviceSubnets)}
+    serviceSubnets: ${format("%#v",split(",",serviceSubnets))}
   proxy:
     mode: ipvs
   apiServer:
@@ -50,7 +68,9 @@ cluster:
         node-cidr-mask-size-ipv6: 112
   scheduler: {}
   etcd: {}
-  extraManifests:
+  externalCloudProvider:
-    - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/kubelet-serving-cert-approver.yaml
+    enabled: true
-    - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/metrics-server.yaml
+    manifests:
-    - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/local-path-storage.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/kubelet-serving-cert-approver.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/metrics-server.yaml
+      - https://raw.githubusercontent.com/sergelogvinov/terraform-talos/main/scaleway/deployments/local-path-storage.yaml

scaleway/variables.tf

@@ -26,11 +26,14 @@ variable "regions" {
 variable "kubernetes" {
   type = map(string)
   default = {
-    podSubnets     = "10.32.0.0/12,f00d:10:32::/102"
+    podSubnets     = "10.32.0.0/12,fd40:10:32::/102"
     serviceSubnets = "10.200.0.0/22,fd40:10:200::/112"
+    nodeSubnets    = "192.168.0.0/16"
     domain         = "cluster.local"
-    cluster_name   = "talos-k8s-scaleway"
+    apiDomain      = "api.cluster.local"
-    tokenmachine   = ""
+    clusterName    = "talos-k8s-hetzner"
+    tokenMachine   = ""
+    caMachine      = ""
     token          = ""
     ca             = ""
   }
@@ -53,27 +56,13 @@ variable "controlplane" {
 }
 
 variable "instances" {
-  description = "Map of region's properties"
+  description = "Map of instance properties"
   type        = map(any)
   default = {
-    "nbg1" = {
+    web_count            = 0,
-      web_count            = 0,
+    web_instance_type    = "DEV1-L",
-      web_instance_type    = "DEV1-L",
+    worker_count         = 0,
-      worker_count         = 0,
+    worker_instance_type = "DEV1-L",
-      worker_instance_type = "DEV1-L",
-    },
-    "fsn1" = {
-      web_count            = 0,
-      web_instance_type    = "DEV1-L",
-      worker_count         = 0,
-      worker_instance_type = "DEV1-L",
-    }
-    "hel1" = {
-      web_count            = 0,
-      web_instance_type    = "DEV1-L",
-      worker_count         = 0,
-      worker_instance_type = "DEV1-L",
-    }
   }
 }
 

scaleway/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     scaleway = {
       source  = "scaleway/scaleway"
-      version = "~> 2.1.0"
+      version = "~> 2.2.0"
     }
   }
   required_version = ">= 1.0"

system_os/.gitignore

@@ -0,0 +1 @@
+*.raw.xz

system_os/scaleway/scaleway.pkr.hcl

@@ -42,7 +42,7 @@ build {
   sources = ["source.scaleway.talos"]
 
   provisioner "file" {
-    source      = "../../../talos/_out/scaleway-amd64.raw.xz"
+    source      = "scaleway-amd64.raw.xz"
     destination = "/tmp/talos.raw.xz"
   }
   provisioner "shell" {

system_os/scaleway/variables.pkr.hcl

@@ -23,7 +23,7 @@ variable "scaleway_zone" {
 
 variable "talos_version" {
   type    = string
-  default = "v0.14.0"
+  default = "v0.14.1"
 }
 
 locals {