From 8059eb9f0cdd4cc7c0d0d2bfe88b44298ee177d9 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 27 Dec 2024 20:38:36 -0800 Subject: [PATCH] Remove support for Calico CNI * Cilium has been the default for about 3 years and is the defacto standard CNI choice. flannel is supported as a simple alternative * Remove various historical options that were needed that are specific to Calico --- CHANGES.md | 3 +++ README.md | 2 +- aws/fedora-coreos/kubernetes/README.md | 2 +- aws/fedora-coreos/kubernetes/bootstrap.tf | 3 +-- .../kubernetes/butane/controller.yaml | 2 -- aws/fedora-coreos/kubernetes/variables.tf | 8 +------- .../kubernetes/workers/butane/worker.yaml | 2 -- aws/flatcar-linux/kubernetes/README.md | 2 +- aws/flatcar-linux/kubernetes/bootstrap.tf | 3 +-- .../kubernetes/butane/controller.yaml | 2 -- aws/flatcar-linux/kubernetes/variables.tf | 8 +------- .../kubernetes/workers/butane/worker.yaml | 2 -- azure/fedora-coreos/kubernetes/README.md | 2 +- azure/fedora-coreos/kubernetes/bootstrap.tf | 9 ++------- .../kubernetes/butane/controller.yaml | 2 -- azure/fedora-coreos/kubernetes/variables.tf | 2 +- .../kubernetes/workers/butane/worker.yaml | 2 -- azure/flatcar-linux/kubernetes/README.md | 2 +- azure/flatcar-linux/kubernetes/bootstrap.tf | 9 ++------- .../kubernetes/butane/controller.yaml | 2 -- azure/flatcar-linux/kubernetes/variables.tf | 2 +- .../kubernetes/workers/butane/worker.yaml | 2 -- bare-metal/fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 18 ++++++++---------- .../kubernetes/butane/controller.yaml | 2 -- .../fedora-coreos/kubernetes/variables.tf | 14 +------------- .../kubernetes/worker/butane/worker.yaml | 2 -- bare-metal/flatcar-linux/kubernetes/README.md | 2 +- .../flatcar-linux/kubernetes/bootstrap.tf | 18 ++++++++---------- .../kubernetes/butane/controller.yaml | 2 -- .../flatcar-linux/kubernetes/variables.tf | 14 +------------- .../kubernetes/worker/butane/worker.yaml | 2 -- .../fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 8 ++------ .../kubernetes/butane/controller.yaml | 2 -- .../kubernetes/butane/worker.yaml | 2 -- .../fedora-coreos/kubernetes/network.tf | 2 +- .../fedora-coreos/kubernetes/variables.tf | 2 +- .../flatcar-linux/kubernetes/README.md | 2 +- .../flatcar-linux/kubernetes/bootstrap.tf | 8 ++------ .../kubernetes/butane/controller.yaml | 2 -- .../kubernetes/butane/worker.yaml | 2 -- .../flatcar-linux/kubernetes/network.tf | 2 +- .../flatcar-linux/kubernetes/variables.tf | 2 +- docs/addons/overview.md | 5 ++--- docs/advanced/nodes.md | 4 ++-- docs/architecture/concepts.md | 2 +- docs/architecture/operating-systems.md | 2 +- docs/fedora-coreos/aws.md | 8 ++------ docs/fedora-coreos/azure.md | 4 ++-- docs/fedora-coreos/bare-metal.md | 7 ++----- docs/fedora-coreos/digitalocean.md | 4 ++-- docs/fedora-coreos/google-cloud.md | 4 ++-- docs/flatcar-linux/aws.md | 8 ++------ docs/flatcar-linux/azure.md | 4 ++-- docs/flatcar-linux/bare-metal.md | 7 ++----- docs/flatcar-linux/digitalocean.md | 4 ++-- docs/flatcar-linux/google-cloud.md | 4 ++-- docs/index.md | 8 ++++---- docs/topics/maintenance.md | 2 +- docs/topics/performance.md | 2 +- docs/topics/security.md | 6 ++---- .../fedora-coreos/kubernetes/README.md | 2 +- .../fedora-coreos/kubernetes/bootstrap.tf | 3 +-- .../kubernetes/butane/controller.yaml | 2 -- .../fedora-coreos/kubernetes/variables.tf | 2 +- .../kubernetes/workers/butane/worker.yaml | 2 -- .../flatcar-linux/kubernetes/README.md | 2 +- .../flatcar-linux/kubernetes/bootstrap.tf | 3 +-- .../kubernetes/butane/controller.yaml | 2 -- .../flatcar-linux/kubernetes/variables.tf | 2 +- .../kubernetes/workers/butane/worker.yaml | 2 -- 72 files changed, 88 insertions(+), 204 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 02491d44..8529961c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,9 @@ Notable changes between versions. * Kubernetes [v1.32.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.32.md#v1320) * Change the default Pod CIDR from 10.2.0.0/16 to 10.20.0.0/14 ([#1555](https://github.com/poseidon/typhoon/pull/1555)) * Configure Kubelets for parallel image pulls ([#1556](https://github.com/poseidon/typhoon/pull/1556)) +* Remove support for Calico CNI (choose between `networking` cilium or flannel) ([#1558](https://github.com/poseidon/typhoon/pull/1558)) + * Remove `network_mtu`, `network_encapsulation`, and `network_ip_autodetection_method` variables (Calico-specific) + * Remove Calico-specific Kubelet mounts # v1.31.4 diff --git a/README.md b/README.md index a018731b..7a64f594 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/flatcar-linux/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/aws/fedora-coreos/kubernetes/README.md b/aws/fedora-coreos/kubernetes/README.md index 14a60140..34221b17 100644 --- a/aws/fedora-coreos/kubernetes/README.md +++ b/aws/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/fedora-coreos/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/aws/fedora-coreos/kubernetes/bootstrap.tf b/aws/fedora-coreos/kubernetes/bootstrap.tf index 3de91e62..98d561f1 100644 --- a/aws/fedora-coreos/kubernetes/bootstrap.tf +++ b/aws/fedora-coreos/kubernetes/bootstrap.tf @@ -1,12 +1,11 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = aws_route53_record.etcds.*.fqdn networking = var.networking - network_mtu = var.network_mtu pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/aws/fedora-coreos/kubernetes/butane/controller.yaml b/aws/fedora-coreos/kubernetes/butane/controller.yaml index 03197052..00475925 100644 --- a/aws/fedora-coreos/kubernetes/butane/controller.yaml +++ b/aws/fedora-coreos/kubernetes/butane/controller.yaml @@ -62,7 +62,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -80,7 +79,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/aws/fedora-coreos/kubernetes/variables.tf b/aws/fedora-coreos/kubernetes/variables.tf index d205bf8c..9194cb6b 100644 --- a/aws/fedora-coreos/kubernetes/variables.tf +++ b/aws/fedora-coreos/kubernetes/variables.tf @@ -133,16 +133,10 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } -variable "network_mtu" { - type = number - description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - default = 1480 -} - variable "host_cidr" { type = string description = "CIDR IPv4 range to assign to EC2 nodes" diff --git a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml index c28ad4d1..49f92e8c 100644 --- a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -34,7 +34,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -52,7 +51,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/aws/flatcar-linux/kubernetes/README.md b/aws/flatcar-linux/kubernetes/README.md index fb88f9fc..6886d24e 100644 --- a/aws/flatcar-linux/kubernetes/README.md +++ b/aws/flatcar-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot](https://typhoon.psdn.io/flatcar-linux/aws/#spot) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/aws/flatcar-linux/kubernetes/bootstrap.tf b/aws/flatcar-linux/kubernetes/bootstrap.tf index 3de91e62..98d561f1 100644 --- a/aws/flatcar-linux/kubernetes/bootstrap.tf +++ b/aws/flatcar-linux/kubernetes/bootstrap.tf @@ -1,12 +1,11 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = aws_route53_record.etcds.*.fqdn networking = var.networking - network_mtu = var.network_mtu pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/aws/flatcar-linux/kubernetes/butane/controller.yaml b/aws/flatcar-linux/kubernetes/butane/controller.yaml index f5fdb62a..afb7769f 100644 --- a/aws/flatcar-linux/kubernetes/butane/controller.yaml +++ b/aws/flatcar-linux/kubernetes/butane/controller.yaml @@ -63,7 +63,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=/usr/bin/docker run -d \ @@ -78,7 +77,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/aws/flatcar-linux/kubernetes/variables.tf b/aws/flatcar-linux/kubernetes/variables.tf index 9dce6843..20dd2d34 100644 --- a/aws/flatcar-linux/kubernetes/variables.tf +++ b/aws/flatcar-linux/kubernetes/variables.tf @@ -133,16 +133,10 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } -variable "network_mtu" { - type = number - description = "CNI interface MTU (applies to calico only). Use 8981 if using instances types with Jumbo frames." - default = 1480 -} - variable "host_cidr" { type = string description = "CIDR IPv4 range to assign to EC2 nodes" diff --git a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml index 2805a68e..fc633733 100644 --- a/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/aws/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -35,7 +35,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" # Podman, rkt, or runc run container processes, whereas docker run @@ -53,7 +52,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/azure/fedora-coreos/kubernetes/README.md b/azure/fedora-coreos/kubernetes/README.md index 8c965a68..de6c9461 100644 --- a/azure/fedora-coreos/kubernetes/README.md +++ b/azure/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [spot priority](https://typhoon.psdn.io/fedora-coreos/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/azure/fedora-coreos/kubernetes/bootstrap.tf b/azure/fedora-coreos/kubernetes/bootstrap.tf index 6027c053..4b723e37 100644 --- a/azure/fedora-coreos/kubernetes/bootstrap.tf +++ b/azure/fedora-coreos/kubernetes/bootstrap.tf @@ -1,17 +1,12 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) - networking = var.networking - # only effective with Calico networking - # we should be able to use 1450 MTU, but in practice, 1410 was needed - network_encapsulation = "vxlan" - network_mtu = "1410" - + networking = var.networking pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/azure/fedora-coreos/kubernetes/butane/controller.yaml b/azure/fedora-coreos/kubernetes/butane/controller.yaml index 6fbcb132..9beb852d 100644 --- a/azure/fedora-coreos/kubernetes/butane/controller.yaml +++ b/azure/fedora-coreos/kubernetes/butane/controller.yaml @@ -58,7 +58,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -76,7 +75,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/azure/fedora-coreos/kubernetes/variables.tf b/azure/fedora-coreos/kubernetes/variables.tf index 52ff4a07..28978bc1 100644 --- a/azure/fedora-coreos/kubernetes/variables.tf +++ b/azure/fedora-coreos/kubernetes/variables.tf @@ -114,7 +114,7 @@ variable "azure_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml index 93e5783a..21dca19e 100644 --- a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -30,7 +30,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -48,7 +47,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/azure/flatcar-linux/kubernetes/README.md b/azure/flatcar-linux/kubernetes/README.md index 23df7651..7181a621 100644 --- a/azure/flatcar-linux/kubernetes/README.md +++ b/azure/flatcar-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [low-priority](https://typhoon.psdn.io/flatcar-linux/azure/#low-priority) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/azure/flatcar-linux/kubernetes/bootstrap.tf b/azure/flatcar-linux/kubernetes/bootstrap.tf index 6027c053..4b723e37 100644 --- a/azure/flatcar-linux/kubernetes/bootstrap.tf +++ b/azure/flatcar-linux/kubernetes/bootstrap.tf @@ -1,17 +1,12 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = formatlist("%s.%s", azurerm_dns_a_record.etcds.*.name, var.dns_zone) - networking = var.networking - # only effective with Calico networking - # we should be able to use 1450 MTU, but in practice, 1410 was needed - network_encapsulation = "vxlan" - network_mtu = "1410" - + networking = var.networking pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/azure/flatcar-linux/kubernetes/butane/controller.yaml b/azure/flatcar-linux/kubernetes/butane/controller.yaml index 8dc95e4f..661ff6e4 100644 --- a/azure/flatcar-linux/kubernetes/butane/controller.yaml +++ b/azure/flatcar-linux/kubernetes/butane/controller.yaml @@ -60,7 +60,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=/usr/bin/docker run -d \ @@ -75,7 +74,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/azure/flatcar-linux/kubernetes/variables.tf b/azure/flatcar-linux/kubernetes/variables.tf index 6fde9c8d..b9584f54 100644 --- a/azure/flatcar-linux/kubernetes/variables.tf +++ b/azure/flatcar-linux/kubernetes/variables.tf @@ -120,7 +120,7 @@ variable "azure_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml index 2e0ce82a..b05f64e0 100644 --- a/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/azure/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -32,7 +32,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" # Podman, rkt, or runc run container processes, whereas docker run @@ -50,7 +49,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/bare-metal/fedora-coreos/kubernetes/README.md b/bare-metal/fedora-coreos/kubernetes/README.md index 1466d34c..79b35c51 100644 --- a/bare-metal/fedora-coreos/kubernetes/README.md +++ b/bare-metal/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf index 953acca7..0f88f5b1 100644 --- a/bare-metal/fedora-coreos/kubernetes/bootstrap.tf +++ b/bare-metal/fedora-coreos/kubernetes/bootstrap.tf @@ -1,16 +1,14 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" - cluster_name = var.cluster_name - api_servers = [var.k8s_domain_name] - etcd_servers = var.controllers.*.domain - networking = var.networking - network_mtu = var.network_mtu - network_ip_autodetection_method = var.network_ip_autodetection_method - pod_cidr = var.pod_cidr - service_cidr = var.service_cidr - components = var.components + cluster_name = var.cluster_name + api_servers = [var.k8s_domain_name] + etcd_servers = var.controllers.*.domain + networking = var.networking + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + components = var.components } diff --git a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml index fcd19a0f..03b08f76 100644 --- a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml +++ b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml @@ -57,7 +57,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -75,7 +74,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/bare-metal/fedora-coreos/kubernetes/variables.tf b/bare-metal/fedora-coreos/kubernetes/variables.tf index 2e2ef362..068ea179 100644 --- a/bare-metal/fedora-coreos/kubernetes/variables.tf +++ b/bare-metal/fedora-coreos/kubernetes/variables.tf @@ -88,22 +88,10 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } -variable "network_mtu" { - type = number - description = "CNI interface MTU (applies to calico only)" - default = 1480 -} - -variable "network_ip_autodetection_method" { - type = string - description = "Method to autodetect the host IPv4 address (applies to calico only)" - default = "first-found" -} - variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods" diff --git a/bare-metal/fedora-coreos/kubernetes/worker/butane/worker.yaml b/bare-metal/fedora-coreos/kubernetes/worker/butane/worker.yaml index 313bf7cb..ee65bd9b 100644 --- a/bare-metal/fedora-coreos/kubernetes/worker/butane/worker.yaml +++ b/bare-metal/fedora-coreos/kubernetes/worker/butane/worker.yaml @@ -29,7 +29,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -47,7 +46,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/bare-metal/flatcar-linux/kubernetes/README.md b/bare-metal/flatcar-linux/kubernetes/README.md index 9a1c8b6e..f3c1fd71 100644 --- a/bare-metal/flatcar-linux/kubernetes/README.md +++ b/bare-metal/flatcar-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/bare-metal/flatcar-linux/kubernetes/bootstrap.tf b/bare-metal/flatcar-linux/kubernetes/bootstrap.tf index 62df3fb6..6c253f2d 100644 --- a/bare-metal/flatcar-linux/kubernetes/bootstrap.tf +++ b/bare-metal/flatcar-linux/kubernetes/bootstrap.tf @@ -1,15 +1,13 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" - cluster_name = var.cluster_name - api_servers = [var.k8s_domain_name] - etcd_servers = var.controllers.*.domain - networking = var.networking - network_mtu = var.network_mtu - network_ip_autodetection_method = var.network_ip_autodetection_method - pod_cidr = var.pod_cidr - service_cidr = var.service_cidr - components = var.components + cluster_name = var.cluster_name + api_servers = [var.k8s_domain_name] + etcd_servers = var.controllers.*.domain + networking = var.networking + pod_cidr = var.pod_cidr + service_cidr = var.service_cidr + components = var.components } diff --git a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml index a8bba41b..8008e683 100644 --- a/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml +++ b/bare-metal/flatcar-linux/kubernetes/butane/controller.yaml @@ -68,7 +68,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=/usr/bin/docker run -d \ @@ -83,7 +82,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/bare-metal/flatcar-linux/kubernetes/variables.tf b/bare-metal/flatcar-linux/kubernetes/variables.tf index 4fafcb05..3b21cb4b 100644 --- a/bare-metal/flatcar-linux/kubernetes/variables.tf +++ b/bare-metal/flatcar-linux/kubernetes/variables.tf @@ -87,22 +87,10 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } -variable "network_mtu" { - type = number - description = "CNI interface MTU (applies to calico only)" - default = 1480 -} - -variable "network_ip_autodetection_method" { - type = string - description = "Method to autodetect the host IPv4 address (applies to calico only)" - default = "first-found" -} - variable "pod_cidr" { type = string description = "CIDR IPv4 range to assign Kubernetes pods" diff --git a/bare-metal/flatcar-linux/kubernetes/worker/butane/worker.yaml b/bare-metal/flatcar-linux/kubernetes/worker/butane/worker.yaml index e537dcb1..1886b116 100644 --- a/bare-metal/flatcar-linux/kubernetes/worker/butane/worker.yaml +++ b/bare-metal/flatcar-linux/kubernetes/worker/butane/worker.yaml @@ -40,7 +40,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" # Podman, rkt, or runc run container processes, whereas docker run @@ -58,7 +57,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/digital-ocean/fedora-coreos/kubernetes/README.md b/digital-ocean/fedora-coreos/kubernetes/README.md index 544812a8..990aa5bf 100644 --- a/digital-ocean/fedora-coreos/kubernetes/README.md +++ b/digital-ocean/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf index 586beda6..39af6b06 100644 --- a/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf +++ b/digital-ocean/fedora-coreos/kubernetes/bootstrap.tf @@ -1,16 +1,12 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = digitalocean_record.etcds.*.fqdn - networking = var.networking - # only effective with Calico networking - network_encapsulation = "vxlan" - network_mtu = "1450" - + networking = var.networking pod_cidr = var.pod_cidr service_cidr = var.service_cidr components = var.components diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml index 7222ca6c..9e130f53 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml @@ -60,7 +60,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -78,7 +77,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml index 53288839..003974b3 100644 --- a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml +++ b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml @@ -33,7 +33,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -51,7 +50,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/digital-ocean/fedora-coreos/kubernetes/network.tf b/digital-ocean/fedora-coreos/kubernetes/network.tf index b506c64e..f65a3a41 100644 --- a/digital-ocean/fedora-coreos/kubernetes/network.tf +++ b/digital-ocean/fedora-coreos/kubernetes/network.tf @@ -39,7 +39,7 @@ resource "digitalocean_firewall" "rules" { source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] } - # IANA vxlan (flannel, calico) + # IANA vxlan (flannel) inbound_rule { protocol = "udp" port_range = "4789" diff --git a/digital-ocean/fedora-coreos/kubernetes/variables.tf b/digital-ocean/fedora-coreos/kubernetes/variables.tf index 56a1db92..6954b83e 100644 --- a/digital-ocean/fedora-coreos/kubernetes/variables.tf +++ b/digital-ocean/fedora-coreos/kubernetes/variables.tf @@ -67,7 +67,7 @@ variable "ssh_fingerprints" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/digital-ocean/flatcar-linux/kubernetes/README.md b/digital-ocean/flatcar-linux/kubernetes/README.md index af32c58a..92073a12 100644 --- a/digital-ocean/flatcar-linux/kubernetes/README.md +++ b/digital-ocean/flatcar-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf b/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf index 586beda6..39af6b06 100644 --- a/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf +++ b/digital-ocean/flatcar-linux/kubernetes/bootstrap.tf @@ -1,16 +1,12 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = digitalocean_record.etcds.*.fqdn - networking = var.networking - # only effective with Calico networking - network_encapsulation = "vxlan" - network_mtu = "1450" - + networking = var.networking pod_cidr = var.pod_cidr service_cidr = var.service_cidr components = var.components diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml index 71e44aa1..7269e8e0 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/controller.yaml @@ -71,7 +71,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=/usr/bin/docker run -d \ @@ -86,7 +85,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml index b265f4e9..e3a893f3 100644 --- a/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml +++ b/digital-ocean/flatcar-linux/kubernetes/butane/worker.yaml @@ -43,7 +43,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" # Podman, rkt, or runc run container processes, whereas docker run @@ -61,7 +60,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/digital-ocean/flatcar-linux/kubernetes/network.tf b/digital-ocean/flatcar-linux/kubernetes/network.tf index b506c64e..f65a3a41 100644 --- a/digital-ocean/flatcar-linux/kubernetes/network.tf +++ b/digital-ocean/flatcar-linux/kubernetes/network.tf @@ -39,7 +39,7 @@ resource "digitalocean_firewall" "rules" { source_tags = [digitalocean_tag.controllers.name, digitalocean_tag.workers.name] } - # IANA vxlan (flannel, calico) + # IANA vxlan (flannel) inbound_rule { protocol = "udp" port_range = "4789" diff --git a/digital-ocean/flatcar-linux/kubernetes/variables.tf b/digital-ocean/flatcar-linux/kubernetes/variables.tf index 23b1f0c1..b9df5aaa 100644 --- a/digital-ocean/flatcar-linux/kubernetes/variables.tf +++ b/digital-ocean/flatcar-linux/kubernetes/variables.tf @@ -67,7 +67,7 @@ variable "ssh_fingerprints" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/docs/addons/overview.md b/docs/addons/overview.md index ac6680d6..5b54bd8c 100644 --- a/docs/addons/overview.md +++ b/docs/addons/overview.md @@ -2,7 +2,7 @@ Typhoon's component model allows for managing cluster components independent from the cluster's lifecycle, upgrading in a rolling or automated fashion, or customizing components in advanced ways. -Typhoon clusters install core components like `CoreDNS`, `kube-proxy`, and a chosen CNI provider (`flannel`, `calico`, or `cilium`) by default. Since v1.30.1, pre-installed components are optional. Other "addon" components like Nginx Ingress, Prometheus, or Grafana may be optionally applied though the component model (after cluster creation). +Typhoon clusters install core components like `CoreDNS`, `kube-proxy`, and a chosen CNI provider (`flannel` or `cilium`) by default. Since v1.30.1, pre-installed components are optional. Other "addon" components like Nginx Ingress, Prometheus, or Grafana may be optionally applied though the component model (after cluster creation). ## Components @@ -12,7 +12,6 @@ Pre-installed by default: * kube-proxy * CNI provider (set via `var.networking`) * flannel - * Calico * Cilium Addons: @@ -24,7 +23,7 @@ Addons: ## Pre-installed Components -By default, Typhoon clusters install `CoreDNS`, `kube-proxy`, and a chosen CNI provider (`flannel`, `calico`, or `cilium`). Disable any or all of these components using the `components` system. +By default, Typhoon clusters install `CoreDNS`, `kube-proxy`, and a chosen CNI provider (`flannel` or `cilium`). Disable any or all of these components using the `components` system. ```tf module "yavin" { diff --git a/docs/advanced/nodes.md b/docs/advanced/nodes.md index 57b86a43..d0951ba1 100644 --- a/docs/advanced/nodes.md +++ b/docs/advanced/nodes.md @@ -83,7 +83,7 @@ In the example above, the two default workers would be labeled `pool: default` a Add custom initial taints on worker pool nodes to indicate a node is unique and should only schedule workloads that explicitly tolerate a given taint key. !!! warning - Since taints prevent workloads scheduling onto a node, you must decide whether `kube-system` DaemonSets (e.g. flannel, Calico, Cilium) should tolerate your custom taint by setting `daemonset_tolerations`. If you don't list your custom taint(s), important components won't run on these nodes. + Since taints prevent workloads scheduling onto a node, you must decide whether `kube-system` DaemonSets (e.g. flannel, Cilium) should tolerate your custom taint by setting `daemonset_tolerations`. If you don't list your custom taint(s), important components won't run on these nodes. === "Cluster" @@ -130,5 +130,5 @@ Add custom initial taints on worker pool nodes to indicate a node is unique and } ``` -In the example above, the the additional worker would be tainted with `role=gpu:NoSchedule` to prevent workloads scheduling, but `kube-system` components like flannel, Calico, or Cilium would tolerate that custom taint to run there. +In the example above, the the additional worker would be tainted with `role=gpu:NoSchedule` to prevent workloads scheduling, but `kube-system` components like flannel or Cilium would tolerate that custom taint to run there. diff --git a/docs/architecture/concepts.md b/docs/architecture/concepts.md index ccec89b1..729f29f7 100644 --- a/docs/architecture/concepts.md +++ b/docs/architecture/concepts.md @@ -8,7 +8,7 @@ Let's cover the concepts you'll need to get started. #### Nodes -All cluster nodes provision themselves from a declarative configuration upfront. Nodes run a `kubelet` service and register themselves with the control plane to join the cluster. All nodes run `kube-proxy` and `calico` or `flannel` pods. +All cluster nodes provision themselves from a declarative configuration upfront. Nodes run a `kubelet` service and register themselves with the control plane to join the cluster. All nodes run `kube-proxy` and `cilium` or `flannel` pods. #### Controllers diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 4de58b94..38d187e1 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -39,7 +39,7 @@ Together, they diversify Typhoon to support a range of container technologies. | control plane images | upstream images | upstream images | | on-host etcd | docker | podman | | on-host kubelet | docker | podman | -| CNI plugins | calico, cilium, flannel | calico, cilium, flannel | +| CNI plugins | cilium, flannel | cilium, flannel | | coordinated drain & OS update | [FLUO](https://github.com/kinvolk/flatcar-linux-update-operator) addon | [fleetlock](https://github.com/poseidon/fleetlock) | ## Directory Locations diff --git a/docs/fedora-coreos/aws.md b/docs/fedora-coreos/aws.md index 8c5c15b8..38f1028e 100644 --- a/docs/fedora-coreos/aws.md +++ b/docs/fedora-coreos/aws.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on AWS with Fedora C We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -224,8 +224,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | | controller_snippets | Controller Butane snippets | [] | [examples](/advanced/customization/) | | worker_snippets | Worker Butane snippets | [] | [examples](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | -| network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | @@ -236,9 +235,6 @@ Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-typ !!! warning Do not choose a `controller_type` smaller than `t3.small`. Smaller instances are not sufficient for running a controller. -!!! tip "MTU" - If your EC2 instance type supports [Jumbo frames](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) (most do), we recommend you change the `network_mtu` to 8981! You will get better pod-to-pod bandwidth. - #### Spot Add `worker_price = "0.10"` to use spot instance workers (instead of "on-demand") and set a maximum spot price in USD. Clusters can tolerate spot market interuptions fairly well (reschedules pods, but cannot drain) to save money, with the tradeoff that requests for workers may go unfulfilled. diff --git a/docs/fedora-coreos/azure.md b/docs/fedora-coreos/azure.md index 7d1f6ccd..34c0e601 100644 --- a/docs/fedora-coreos/azure.md +++ b/docs/fedora-coreos/azure.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on Azure with Fedora We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -252,7 +252,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Butane snippets | [] | [example](/advanced/customization/#usage) | | worker_snippets | Worker Butane snippets | [] | [example](/advanced/customization/#usage) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | network_cidr | Virtual network CIDR ranges | { ipv4 = ["10.0.0.0/16"], ipv6 = [ULA, ...] } | { ipv4 = ["10.0.0.0/20"] } | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/fedora-coreos/bare-metal.md b/docs/fedora-coreos/bare-metal.md index fc4c1c6a..0ecf4855 100644 --- a/docs/fedora-coreos/bare-metal.md +++ b/docs/fedora-coreos/bare-metal.md @@ -4,7 +4,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.32.0 cluster First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Fedora CoreOS to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -292,7 +292,6 @@ $ journalctl -f -u bootstrap podman[1750]: The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? podman[1750]: Waiting for static pod control plane ... -podman[1750]: serviceaccount/calico-node unchanged systemd[1]: Started Kubernetes control plane. ``` @@ -365,10 +364,8 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | workers | List of worker machine detail objects (unique name, identifying MAC address, FQDN) | [] | `[{name="node2", mac="52:54:00:b2:2f:86", domain="node2.example.com"}, {name="node3", mac="52:54:00:c3:61:77", domain="node3.example.com"}]` | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Fedora CoreOS images into the cache | false | true | | install_disk | Disk device where Fedora CoreOS should be installed | "sda" (not "/dev/sda" like Container Linux) | "sdb" | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | -| network_mtu | CNI interface MTU (calico-only) | 1480 | - | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | snippets | Map from machine names to lists of Butane snippets | {} | [examples](/advanced/customization/) | -| network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | diff --git a/docs/fedora-coreos/digitalocean.md b/docs/fedora-coreos/digitalocean.md index f3563029..aef57534 100644 --- a/docs/fedora-coreos/digitalocean.md +++ b/docs/fedora-coreos/digitalocean.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on DigitalOcean with We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -241,7 +241,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | | controller_snippets | Controller Butane snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Butane snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/fedora-coreos/google-cloud.md b/docs/fedora-coreos/google-cloud.md index d5e57ba6..a0fc9e04 100644 --- a/docs/fedora-coreos/google-cloud.md +++ b/docs/fedora-coreos/google-cloud.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on Google Compute En We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -224,7 +224,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Butane snippets | [] | [examples](/advanced/customization/) | | worker_snippets | Worker Butane snippets | [] | [examples](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | diff --git a/docs/flatcar-linux/aws.md b/docs/flatcar-linux/aws.md index e4822796..e451ce86 100644 --- a/docs/flatcar-linux/aws.md +++ b/docs/flatcar-linux/aws.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on AWS with Flatcar We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a VPC, gateway, subnets, security groups, controller instances, worker auto-scaling group, network load balancer, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -222,8 +222,7 @@ Reference the DNS zone id with `aws_route53_zone.zone-for-clusters.zone_id`. | worker_target_groups | Target group ARNs to which worker instances should be added | [] | [aws_lb_target_group.app.id] | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | -| network_mtu | CNI interface MTU (calico only) | 1480 | 8981 | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | host_cidr | CIDR IPv4 range to assign to EC2 instances | "10.0.0.0/16" | "10.1.0.0/16" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | @@ -234,9 +233,6 @@ Check the list of valid [instance types](https://aws.amazon.com/ec2/instance-typ !!! warning Do not choose a `controller_type` smaller than `t3.small`. Smaller instances are not sufficient for running a controller. -!!! tip "MTU" - If your EC2 instance type supports [Jumbo frames](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) (most do), we recommend you change the `network_mtu` to 8981! You will get better pod-to-pod bandwidth. - #### Spot Add `worker_price = "0.10"` to use spot instance workers (instead of "on-demand") and set a maximum spot price in USD. Clusters can tolerate spot market interuptions fairly well (reschedules pods, but cannot drain) to save money, with the tradeoff that requests for workers may go unfulfilled. diff --git a/docs/flatcar-linux/azure.md b/docs/flatcar-linux/azure.md index 662f50ff..dac53169 100644 --- a/docs/flatcar-linux/azure.md +++ b/docs/flatcar-linux/azure.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on Azure with Flatca We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a resource group, virtual network, subnets, security groups, controller availability set, worker scale set, load balancer, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -240,7 +240,7 @@ Reference the DNS zone with `azurerm_dns_zone.clusters.name` and its resource gr | worker_priority | Set priority to Spot to use reduced cost surplus capacity, with the tradeoff that instances can be deallocated at any time | Regular | Spot | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/#usage) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | network_cidr | Virtual network CIDR ranges | { ipv4 = ["10.0.0.0/16"], ipv6 = [ULA, ...] } | { ipv4 = ["10.0.0.0/20"] } | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/flatcar-linux/bare-metal.md b/docs/flatcar-linux/bare-metal.md index 5d8cf7a4..77bce46b 100644 --- a/docs/flatcar-linux/bare-metal.md +++ b/docs/flatcar-linux/bare-metal.md @@ -4,7 +4,7 @@ In this tutorial, we'll network boot and provision a Kubernetes v1.32.0 cluster First, we'll deploy a [Matchbox](https://github.com/poseidon/matchbox) service and setup a network boot environment. Then, we'll declare a Kubernetes cluster using the Typhoon Terraform module and power on machines. On PXE boot, machines will install Container Linux to disk, reboot into the disk install, and provision themselves as Kubernetes controllers or workers via Ignition. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns` while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns` while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -302,7 +302,6 @@ $ journalctl -f -u bootstrap The connection to the server cluster.example.com:6443 was refused - did you specify the right host or port? Waiting for static pod control plane ... -serviceaccount/calico-node unchanged systemd[1]: Started Kubernetes control plane. ``` @@ -376,10 +375,8 @@ Check the [variables.tf](https://github.com/poseidon/typhoon/blob/master/bare-me | download_protocol | Protocol iPXE uses to download the kernel and initrd. iPXE must be compiled with [crypto](https://ipxe.org/crypto) support for https. Unused if cached_install is true | "https" | "http" | | cached_install | PXE boot and install from the Matchbox `/assets` cache. Admin MUST have downloaded Container Linux or Flatcar images into the cache | false | true | | install_disk | Disk device where Container Linux should be installed | "/dev/sda" | "/dev/sdb" | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | -| network_mtu | CNI interface MTU (calico-only) | 1480 | - | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | snippets | Map from machine names to lists of Container Linux Config snippets | {} | [examples](/advanced/customization/) | -| network_ip_autodetection_method | Method to detect host IPv4 address (calico-only) | "first-found" | "can-reach=10.0.0.1" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | kernel_args | Additional kernel args to provide at PXE boot | [] | ["kvm-intel.nested=1"] | diff --git a/docs/flatcar-linux/digitalocean.md b/docs/flatcar-linux/digitalocean.md index 3430e774..92b7038e 100644 --- a/docs/flatcar-linux/digitalocean.md +++ b/docs/flatcar-linux/digitalocean.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on DigitalOcean with We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create controller droplets, worker droplets, DNS records, tags, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -241,7 +241,7 @@ Digital Ocean requires the SSH public key be uploaded to your account, so you ma | worker_type | Droplet type for workers | "s-1vcpu-2gb" | s-1vcpu-2gb, s-2vcpu-2gb, ... | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | diff --git a/docs/flatcar-linux/google-cloud.md b/docs/flatcar-linux/google-cloud.md index 18f2f839..96636312 100644 --- a/docs/flatcar-linux/google-cloud.md +++ b/docs/flatcar-linux/google-cloud.md @@ -4,7 +4,7 @@ In this tutorial, we'll create a Kubernetes v1.32.0 cluster on Google Compute En We'll declare a Kubernetes cluster using the Typhoon Terraform module. Then apply the changes to create a network, firewall rules, health checks, controller instances, worker managed instance group, load balancers, and TLS assets. -Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel`, `calico`, or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. +Controller hosts are provisioned to run an `etcd-member` peer and a `kubelet` service. Worker hosts run a `kubelet` service. Controller nodes run `kube-apiserver`, `kube-scheduler`, `kube-controller-manager`, and `coredns`, while `kube-proxy` and (`flannel` or `cilium`) run on every node. A generated `kubeconfig` provides `kubectl` access to the cluster. ## Requirements @@ -222,7 +222,7 @@ resource "google_dns_managed_zone" "zone-for-clusters" { | worker_preemptible | If enabled, Compute Engine will terminate workers randomly within 24 hours | false | true | | controller_snippets | Controller Container Linux Config snippets | [] | [example](/advanced/customization/) | | worker_snippets | Worker Container Linux Config snippets | [] | [example](/advanced/customization/) | -| networking | Choice of networking provider | "cilium" | "calico" or "cilium" or "flannel" | +| networking | Choice of networking provider | "cilium" | "cilium" or "flannel" | | pod_cidr | CIDR IPv4 range to assign to Kubernetes pods | "10.20.0.0/14" | "10.22.0.0/16" | | service_cidr | CIDR IPv4 range to assign to Kubernetes services | "10.3.0.0/16" | "10.3.0.0/24" | | worker_node_labels | List of initial worker node labels | [] | ["worker-pool=default"] | diff --git a/docs/index.md b/docs/index.md index 2a31beb7..8d819dd2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](advanced/worker-pools/), [preemptible](fedora-coreos/google-cloud/#preemption) workers, and [snippets](advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, or other [addons](addons/overview/) @@ -119,9 +119,9 @@ List the pods. ``` $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE -kube-system calico-node-1cs8z 2/2 Running 0 6m -kube-system calico-node-d1l5b 2/2 Running 0 6m -kube-system calico-node-sp9ps 2/2 Running 0 6m +kube-system cilium-1cs8z 2/2 Running 0 6m +kube-system cilium-d1l5b 2/2 Running 0 6m +kube-system cilium-sp9ps 2/2 Running 0 6m kube-system coredns-1187388186-dkh3o 1/1 Running 0 6m kube-system coredns-1187388186-zj5dl 1/1 Running 0 6m kube-system kube-apiserver-controller-0 1/1 Running 0 6m diff --git a/docs/topics/maintenance.md b/docs/topics/maintenance.md index 10b0271e..993c041b 100644 --- a/docs/topics/maintenance.md +++ b/docs/topics/maintenance.md @@ -128,7 +128,7 @@ Apply complete! Resources: 0 added, 0 changed, 55 destroyed. #### In-place Edits -Typhoon uses a static pod Kubernetes control plane which allows certain manifest upgrades to be performed in-place. Components like `kube-apiserver`, `kube-controller-manager`, and `kube-scheduler` are run as static pods. Components `flannel`/`calico`, `coredns`, and `kube-proxy` are scheduled on Kubernetes and can be edited via `kubectl`. +Typhoon uses a static pod Kubernetes control plane which allows certain manifest upgrades to be performed in-place. Components like `kube-apiserver`, `kube-controller-manager`, and `kube-scheduler` are run as static pods. Components `flannel`/`cilium`, `coredns`, and `kube-proxy` are scheduled on Kubernetes and can be edited via `kubectl`. In certain scenarios, in-place edits can be useful for quickly rolling out security patches (e.g. bumping `coredns`) or prioritizing speed over the safety of a proper cluster re-provision and transition. diff --git a/docs/topics/performance.md b/docs/topics/performance.md index 364ac15e..ab5a2388 100644 --- a/docs/topics/performance.md +++ b/docs/topics/performance.md @@ -38,7 +38,7 @@ Network performance varies based on the platform and CNI plugin. `iperf` was use Notes: -* Calico, Cilium, and Flannel have comparable performance. Platform and configuration differences dominate. +* Cilium and Flannel have comparable performance. Platform and configuration differences dominate. * Azure and DigitalOcean network performance can be quite variable or depend on machine type * Only [certain AWS EC2 instance types](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/network_mtu.html#jumbo_frame_instances) allow jumbo frames. This is why the default MTU on AWS must be 1480. diff --git a/docs/topics/security.md b/docs/topics/security.md index bd1eb0fd..2dc666f3 100644 --- a/docs/topics/security.md +++ b/docs/topics/security.md @@ -12,9 +12,9 @@ Typhoon aims to be minimal and secure. We're running it ourselves after all. * [NodeRestriction](https://kubernetes.io/docs/reference/access-authn-authz/node/) is enabled to limit Kubelet authorization * [Role-Based Access Control](https://kubernetes.io/docs/admin/authorization/rbac/) is enabled. Apps must define RBAC policies for API access * Workloads run on worker nodes only, unless they tolerate the master taint -* Kubernetes [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) and Calico [NetworkPolicy](https://docs.projectcalico.org/latest/reference/calicoctl/resources/networkpolicy) support [^1] +* Kubernetes [Network Policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) and Cilium [NetworkPolicy](https://docs.cilium.io/en/latest/security/policy/index.html) support [^1] -[^1]: Requires `networking = "calico"`. Calico is the default on all platforms (AWS, Azure, bare-metal, DigitalOcean, and Google Cloud). +[^1]: Requires `networking = "cilium"`. Cilium is the default on all platforms (AWS, Azure, bare-metal, DigitalOcean, and Google Cloud). **Hosts** @@ -91,7 +91,6 @@ Typhoon publishes Terraform providers to the Terraform Registry, GPG signed by 0 | coredns | NA | false | false | | kube-proxy | root | true | true | | cilium | root | true | true | -| calico | root | true | true | | flannel | root | true | true | @@ -103,7 +102,6 @@ Typhoon publishes Terraform providers to the Terraform Registry, GPG signed by 0 | coredns | system-cluster-critical | | kube-proxy | system-node-critical | | cilium | system-node-critical | -| calico | system-node-critical | | flannel | system-node-critical | ## Disclosures diff --git a/google-cloud/fedora-coreos/kubernetes/README.md b/google-cloud/fedora-coreos/kubernetes/README.md index 9886c1c7..d0f0b4d0 100644 --- a/google-cloud/fedora-coreos/kubernetes/README.md +++ b/google-cloud/fedora-coreos/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/), SELinux enforcing * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/fedora-coreos/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf index 14b2213e..3ea401ef 100644 --- a/google-cloud/fedora-coreos/kubernetes/bootstrap.tf +++ b/google-cloud/fedora-coreos/kubernetes/bootstrap.tf @@ -1,12 +1,11 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")] networking = var.networking - network_mtu = 1440 pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml index afd565d7..8ab40743 100644 --- a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml +++ b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml @@ -58,7 +58,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -76,7 +75,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/google-cloud/fedora-coreos/kubernetes/variables.tf b/google-cloud/fedora-coreos/kubernetes/variables.tf index 00f11f2d..649dfea1 100644 --- a/google-cloud/fedora-coreos/kubernetes/variables.tf +++ b/google-cloud/fedora-coreos/kubernetes/variables.tf @@ -116,7 +116,7 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml index 206de29e..ebc19899 100644 --- a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml @@ -30,7 +30,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=-/usr/bin/podman rm kubelet @@ -48,7 +47,6 @@ systemd: --volume /sys/fs/cgroup:/sys/fs/cgroup \ --volume /etc/selinux:/etc/selinux \ --volume /sys/fs/selinux:/sys/fs/selinux \ - --volume /var/lib/calico:/var/lib/calico:ro \ --volume /var/lib/containerd:/var/lib/containerd \ --volume /var/lib/kubelet:/var/lib/kubelet:rshared,z \ --volume /var/log:/var/log \ diff --git a/google-cloud/flatcar-linux/kubernetes/README.md b/google-cloud/flatcar-linux/kubernetes/README.md index d05eef92..4bb6ef62 100644 --- a/google-cloud/flatcar-linux/kubernetes/README.md +++ b/google-cloud/flatcar-linux/kubernetes/README.md @@ -12,7 +12,7 @@ Typhoon distributes upstream Kubernetes, architectural conventions, and cluster ## Features * Kubernetes v1.32.0 (upstream) -* Single or multi-master, [Calico](https://www.projectcalico.org/) or [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking +* Single or multi-master, [Cilium](https://github.com/cilium/cilium) or [flannel](https://github.com/coreos/flannel) networking * On-cluster etcd with TLS, [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/)-enabled, [network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/) * Advanced features like [worker pools](https://typhoon.psdn.io/advanced/worker-pools/), [preemptible](https://typhoon.psdn.io/flatcar-linux/google-cloud/#preemption) workers, and [snippets](https://typhoon.psdn.io/advanced/customization/#hosts) customization * Ready for Ingress, Prometheus, Grafana, CSI, and other optional [addons](https://typhoon.psdn.io/addons/overview/) diff --git a/google-cloud/flatcar-linux/kubernetes/bootstrap.tf b/google-cloud/flatcar-linux/kubernetes/bootstrap.tf index 14b2213e..3ea401ef 100644 --- a/google-cloud/flatcar-linux/kubernetes/bootstrap.tf +++ b/google-cloud/flatcar-linux/kubernetes/bootstrap.tf @@ -1,12 +1,11 @@ # Kubernetes assets (kubeconfig, manifests) module "bootstrap" { - source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=c775b4de9a16ad1a94fef811f891c49169e7729f" + source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=33f8d2083cd2da5a18f954dd4f765b482d9b8046" cluster_name = var.cluster_name api_servers = [format("%s.%s", var.cluster_name, var.dns_zone)] etcd_servers = [for fqdn in google_dns_record_set.etcds.*.name : trimsuffix(fqdn, ".")] networking = var.networking - network_mtu = 1440 pod_cidr = var.pod_cidr service_cidr = var.service_cidr daemonset_tolerations = var.daemonset_tolerations diff --git a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml index 497d0fc1..98488fd4 100644 --- a/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml +++ b/google-cloud/flatcar-linux/kubernetes/butane/controller.yaml @@ -60,7 +60,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" ExecStartPre=/usr/bin/docker run -d \ @@ -75,7 +74,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \ diff --git a/google-cloud/flatcar-linux/kubernetes/variables.tf b/google-cloud/flatcar-linux/kubernetes/variables.tf index a63bf51d..b1104760 100644 --- a/google-cloud/flatcar-linux/kubernetes/variables.tf +++ b/google-cloud/flatcar-linux/kubernetes/variables.tf @@ -116,7 +116,7 @@ variable "ssh_authorized_key" { variable "networking" { type = string - description = "Choice of networking provider (flannel, calico, or cilium)" + description = "Choice of networking provider (flannel or cilium)" default = "cilium" } diff --git a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml index d27b47db..dc88390f 100644 --- a/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml +++ b/google-cloud/flatcar-linux/kubernetes/workers/butane/worker.yaml @@ -32,7 +32,6 @@ systemd: ExecStartPre=/bin/mkdir -p /etc/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin - ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" # Podman, rkt, or runc run container processes, whereas docker run @@ -50,7 +49,6 @@ systemd: -v /lib/modules:/lib/modules:ro \ -v /run:/run \ -v /sys/fs/cgroup:/sys/fs/cgroup \ - -v /var/lib/calico:/var/lib/calico:ro \ -v /var/lib/containerd:/var/lib/containerd \ -v /var/lib/kubelet:/var/lib/kubelet:rshared \ -v /var/log:/var/log \