Update kube-apiserver runtime-config flags

* MutatingAdmissionPolicy is available as a beta and alpha API

.github/dependabot.yaml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily"

.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  terraform:
+    name: fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v6
+
+      - name: terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.11.1
+
+      - name: fmt
+        run: terraform fmt -check -diff -recursive

README.md

@@ -1,4 +1,7 @@
 # terraform-render-bootstrap
+[![Workflow](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml/badge.svg)](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml?query=branch%3Amain)
+[![Sponsors](https://img.shields.io/github/sponsors/poseidon?logo=github)](https://github.com/sponsors/poseidon)
+[![Mastodon](https://img.shields.io/badge/follow-news-6364ff?logo=mastodon)](https://fosstodon.org/@typhoon)
 
 `terraform-render-bootstrap` is a Terraform module that renders TLS certificates, static pods, and manifests for bootstrapping a Kubernetes cluster.
 
@@ -17,11 +20,10 @@ module "bootstrap" {
   cluster_name = "example"
   api_servers = ["node1.example.com"]
   etcd_servers = ["node1.example.com"]
-  asset_dir = "/home/core/clusters/mycluster"
 }
 ```
 
-Generate the assets.
+Generate assets in Terraform state.
 
 ```sh
 terraform init
@@ -29,5 +31,12 @@ terraform plan
 terraform apply
 ```
 
-Find bootstrap assets rendered to the `asset_dir` path. That's it.
+To inspect and write assets locally (e.g. debugging) use the `assets_dist` Terraform output.
 
+```
+resource local_file "assets" {
+  for_each = module.bootstrap.assets_dist
+  filename = "some-assets/${each.key}"
+  content = each.value
+}
+```

assets.tf

@@ -1,42 +0,0 @@
-# Generated kubeconfig for Kubelets
-data "template_file" "kubeconfig-kubelet" {
-  template = file("${path.module}/resources/kubeconfig-kubelet")
-
-  vars = {
-    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
-    kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem)
-    kubelet_key  = base64encode(tls_private_key.kubelet.private_key_pem)
-    server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
-  }
-}
-
-# Generated admin kubeconfig to bootstrap control plane
-data "template_file" "kubeconfig-admin" {
-  template = file("${path.module}/resources/kubeconfig-admin")
-
-  vars = {
-    name         = var.cluster_name
-    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
-    kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
-    kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
-    server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
-  }
-}
-
-# Generated kubeconfig for Kubelets
-resource "local_file" "kubeconfig-kubelet" {
-  content  = data.template_file.kubeconfig-kubelet.rendered
-  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
-}
-
-# Generated admin kubeconfig to bootstrap control plane
-resource "local_file" "kubeconfig-admin" {
-  content  = data.template_file.kubeconfig-admin.rendered
-  filename = "${var.asset_dir}/auth/kubeconfig"
-}
-
-# Generated admin kubeconfig in a file named after the cluster
-resource "local_file" "kubeconfig-admin-named" {
-  content  = data.template_file.kubeconfig-admin.rendered
-  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
-}

auth.tf

@@ -0,0 +1,68 @@
+locals {
+  # component kubeconfigs assets map
+  auth_kubeconfigs = {
+    "auth/admin.conf"              = local.kubeconfig-admin,
+    "auth/controller-manager.conf" = local.kubeconfig-controller-manager
+    "auth/scheduler.conf"          = local.kubeconfig-scheduler
+  }
+}
+
+locals {
+  # Generated admin kubeconfig to bootstrap control plane
+  kubeconfig-admin = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-controller-manager = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.controller-manager.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.controller-manager.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-scheduler = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.scheduler.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.scheduler.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kubeconfig to bootstrap Kubelets
+  kubeconfig-bootstrap = templatefile("${path.module}/resources/kubeconfig-bootstrap",
+    {
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+      token_id     = random_password.bootstrap-token-id.result
+      token_secret = random_password.bootstrap-token-secret.result
+    }
+  )
+}
+
+# Generate a cryptographically random token id (public)
+resource "random_password" "bootstrap-token-id" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret
+resource "random_password" "bootstrap-token-secret" {
+  length  = 16
+  upper   = false
+  special = false
+}
+

conditional.tf

@@ -1,76 +1,36 @@
 # Assets generated only when certain options are chosen
 
 locals {
-  # flannel manifests (manifest.yaml => content)
+  # flannel manifests map
+  # { manifests-networking/manifest.yaml => content }
   flannel_manifests = {
-    for name in fileset("${path.module}/resources/flannel", "*.yaml"):
-    "manifests-networking/${name}" => templatefile(
+    for name in fileset("${path.module}/resources/flannel", "*.yaml") :
+    "manifests/network/${name}" => templatefile(
       "${path.module}/resources/flannel/${name}",
       {
-        flannel_image     = var.container_images["flannel"]
-        flannel_cni_image = var.container_images["flannel_cni"]
-        pod_cidr          = var.pod_cidr
+        flannel_image         = var.container_images["flannel"]
+        flannel_cni_image     = var.container_images["flannel_cni"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
       }
     )
-    if var.networking == "flannel"
+    if var.components.enable && var.components.flannel.enable && var.networking == "flannel"
   }
 
-  # calico manifests (manifest.yaml => content)
-  calico_manifests = {
-    for name in fileset("${path.module}/resources/calico", "*.yaml"):
-    "manifests-networking/${name}" => templatefile(
-      "${path.module}/resources/calico/${name}",
+  # cilium manifests map
+  # { manifests-networking/manifest.yaml => content }
+  cilium_manifests = {
+    for name in fileset("${path.module}/resources/cilium", "**/*.yaml") :
+    "manifests/network/${name}" => templatefile(
+      "${path.module}/resources/cilium/${name}",
       {
-        calico_image                    = var.container_images["calico"]
-        calico_cni_image                = var.container_images["calico_cni"]
-        network_mtu                     = var.network_mtu
-        network_encapsulation           = indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")
-        ipip_enabled                    = var.network_encapsulation == "ipip" ? true : false
-        ipip_readiness                  = var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""
-        vxlan_enabled                   = var.network_encapsulation == "vxlan" ? true : false
-        network_ip_autodetection_method = var.network_ip_autodetection_method
-        pod_cidr                        = var.pod_cidr
-        enable_reporting                = var.enable_reporting
+        cilium_agent_image    = var.container_images["cilium_agent"]
+        cilium_operator_image = var.container_images["cilium_operator"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
       }
     )
-    if var.networking == "calico"
-  }
-
-  # kube-router manifests (manifest.yaml => content)
-  kube_router_manifests = {
-    for name in fileset("${path.module}/resources/kube-router", "*.yaml"):
-    "manifests-networking/${name}" => templatefile(
-      "${path.module}/resources/kube-router/${name}",
-      {
-        kube_router_image = var.container_images["kube_router"]
-        flannel_cni_image = var.container_images["flannel_cni"]
-        network_mtu       = var.network_mtu
-      }
-    )
-    if var.networking == "kube-router"
+    if var.components.enable && var.components.cilium.enable && var.networking == "cilium"
   }
 }
 
-# flannel manifests
-resource "local_file" "flannel-manifests" {
-  for_each = local.flannel_manifests
-
-  filename = "${var.asset_dir}/${each.key}"
-  content  = each.value
-}
-
-# Calico manifests
-resource "local_file" "calico-manifests" {
-  for_each = local.calico_manifests
-
-  filename = "${var.asset_dir}/${each.key}"
-  content  = each.value
-}
-
-# kube-router manifests
-resource "local_file" "kube-router-manifests" {
-  for_each = local.kube_router_manifests
-
-  filename = "${var.asset_dir}/${each.key}"
-  content  = each.value
-}

manifests.tf

@@ -1,62 +1,74 @@
 locals {
-  # Kubernetes static pod manifests (manifest.yaml => content)
+  # Kubernetes static pod manifests map
+  # {static-manifests/manifest.yaml => content }
   static_manifests = {
-    for name in fileset("${path.module}/resources/static-manifests", "*.yaml"):
+    for name in fileset("${path.module}/resources/static-manifests", "*.yaml") :
     "static-manifests/${name}" => templatefile(
       "${path.module}/resources/static-manifests/${name}",
       {
-        hyperkube_image   = var.container_images["hyperkube"]
-        etcd_servers      = join(",", formatlist("https://%s:2379", var.etcd_servers))
-        cloud_provider    = var.cloud_provider
-        pod_cidr          = var.pod_cidr
-        service_cidr      = var.service_cidr
-        trusted_certs_dir = var.trusted_certs_dir
-        aggregation_flags = var.enable_aggregation ? indent(4, local.aggregation_flags) : ""
+        kube_apiserver_image          = var.container_images["kube_apiserver"]
+        kube_controller_manager_image = var.container_images["kube_controller_manager"]
+        kube_scheduler_image          = var.container_images["kube_scheduler"]
+
+        etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers))
+        pod_cidr     = var.pod_cidr
+        service_cidr = var.service_cidr
+
+        service_account_issuer = var.service_account_issuer
+        aggregation_flags      = var.enable_aggregation ? indent(4, local.aggregation_flags) : ""
       }
     )
   }
 
-  # Kubernetes control plane manifests (manifest.yaml => content)
-  manifests = {
-    for name in fileset("${path.module}/resources/manifests", "**/*.yaml"):
+  # Kubernetes control plane manifests map
+  # { manifests/manifest.yaml => content }
+  manifests = merge({
+    for name in fileset("${path.module}/resources/manifests", "**/*.yaml") :
     "manifests/${name}" => templatefile(
       "${path.module}/resources/manifests/${name}",
       {
-        hyperkube_image        = var.container_images["hyperkube"]
-        coredns_image          = var.container_images["coredns"]
-        control_plane_replicas = max(2, length(var.etcd_servers))
-        pod_cidr               = var.pod_cidr
-        cluster_domain_suffix  = var.cluster_domain_suffix
-        cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
-        trusted_certs_dir      = var.trusted_certs_dir
-        server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        server         = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        apiserver_host = var.api_servers[0]
+        apiserver_port = var.external_apiserver_port
+        token_id       = random_password.bootstrap-token-id.result
+        token_secret   = random_password.bootstrap-token-secret.result
       }
     )
-  }
-}
-
-# Kubernetes static pod manifests
-resource "local_file" "static-manifests" {
-  for_each = local.static_manifests
-
-  content  = each.value
-  filename = "${var.asset_dir}/${each.key}"
-}
-
-# Kubernetes control plane manifests
-resource "local_file" "manifests" {
-  for_each = local.manifests
-
-  content  = each.value
-  filename = "${var.asset_dir}/${each.key}"
+    },
+    # CoreDNS manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/coredns", "*.yaml") :
+      "manifests/coredns/${name}" => templatefile(
+        "${path.module}/resources/coredns/${name}",
+        {
+          coredns_image          = var.container_images["coredns"]
+          control_plane_replicas = max(2, length(var.etcd_servers))
+          cluster_domain_suffix  = var.cluster_domain_suffix
+          cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+        }
+      ) if var.components.enable && var.components.coredns.enable
+    },
+    # kube-proxy manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/kube-proxy", "*.yaml") :
+      "manifests/kube-proxy/${name}" => templatefile(
+        "${path.module}/resources/kube-proxy/${name}",
+        {
+          kube_proxy_image      = var.container_images["kube_proxy"]
+          pod_cidr              = var.pod_cidr
+          daemonset_tolerations = var.daemonset_tolerations
+        }
+      ) if var.components.enable && var.components.kube_proxy.enable && var.networking != "cilium"
+    }
+  )
 }
 
 locals {
   aggregation_flags = <<EOF
 
-- --proxy-client-cert-file=/etc/kubernetes/secrets/aggregation-client.crt
-- --proxy-client-key-file=/etc/kubernetes/secrets/aggregation-client.key
-- --requestheader-client-ca-file=/etc/kubernetes/secrets/aggregation-ca.crt
+- --proxy-client-cert-file=/etc/kubernetes/pki/aggregation-client.crt
+- --proxy-client-key-file=/etc/kubernetes/pki/aggregation-client.key
+- --requestheader-client-ca-file=/etc/kubernetes/pki/aggregation-ca.crt
 - --requestheader-extra-headers-prefix=X-Remote-Extra-
 - --requestheader-group-headers=X-Remote-Group
 - --requestheader-username-headers=X-Remote-User

outputs.tf

@@ -5,40 +5,72 @@ output "cluster_dns_service_ip" {
 
 // Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
 output "kubeconfig-kubelet" {
-  value = data.template_file.kubeconfig-kubelet.rendered
+  value     = local.kubeconfig-bootstrap
+  sensitive = true
 }
 
 // Generated kubeconfig for admins (i.e. human super-user)
 output "kubeconfig-admin" {
-  value = data.template_file.kubeconfig-admin.rendered
+  value     = local.kubeconfig-admin
+  sensitive = true
+}
+
+# assets to distribute to controllers
+# { some/path => content }
+output "assets_dist" {
+  # combine maps of assets
+  value = merge(
+    local.auth_kubeconfigs,
+    local.etcd_tls,
+    local.kubernetes_tls,
+    local.aggregation_tls,
+    local.static_manifests,
+    local.manifests,
+    local.flannel_manifests,
+    local.cilium_manifests,
+  )
+  sensitive = true
 }
 
 # etcd TLS assets
 
 output "etcd_ca_cert" {
-  value = tls_self_signed_cert.etcd-ca.cert_pem
+  value     = tls_self_signed_cert.etcd-ca.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_cert" {
-  value = tls_locally_signed_cert.client.cert_pem
+  value     = tls_locally_signed_cert.client.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_key" {
-  value = tls_private_key.client.private_key_pem
+  value     = tls_private_key.client.private_key_pem
+  sensitive = true
 }
 
 output "etcd_server_cert" {
-  value = tls_locally_signed_cert.server.cert_pem
+  value     = tls_locally_signed_cert.server.cert_pem
+  sensitive = true
 }
 
 output "etcd_server_key" {
-  value = tls_private_key.server.private_key_pem
+  value     = tls_private_key.server.private_key_pem
+  sensitive = true
 }
 
 output "etcd_peer_cert" {
-  value = tls_locally_signed_cert.peer.cert_pem
+  value     = tls_locally_signed_cert.peer.cert_pem
+  sensitive = true
 }
 
 output "etcd_peer_key" {
-  value = tls_private_key.peer.private_key_pem
+  value     = tls_private_key.peer.private_key_pem
+  sensitive = true
+}
+
+# Kubernetes TLS assets
+
+output "service_account_public_key" {
+  value = tls_private_key.service-account.public_key_pem
 }

resources/calico/cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-node
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-node
-subjects:
-- kind: ServiceAccount
-  name: calico-node
-  namespace: kube-system

resources/calico/cluster-role.yaml

@@ -1,109 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-node
-rules:
-  - apiGroups: [""]
-    resources:
-      - pods
-      - nodes
-      - namespaces
-    verbs:
-      - get
-  - apiGroups: [""]
-    resources:
-      - endpoints
-      - services
-    verbs:
-      - watch
-      - list
-  # Used by Calico for policy information
-  - apiGroups: [""]
-    resources:
-      - pods
-      - namespaces
-      - serviceaccounts
-    verbs:
-      - list
-      - watch
-  - apiGroups: [""]
-    resources:
-      - nodes/status
-    verbs:
-      # Calico patches the node NetworkUnavilable status
-      - patch
-      # Calico updates some info in node annotations
-      - update
-  # CNI plugin patches pods/status
-  - apiGroups: [""]
-    resources:
-      - pods/status
-    verbs:
-      - patch
-  # Calico reads some info on nodes
-  - apiGroups: [""]
-    resources:
-      - nodes
-    verbs:
-      - get
-      - list
-      - watch
-  # Calico monitors Kubernetes NetworkPolicies
-  - apiGroups: ["networking.k8s.io"]
-    resources:
-      - networkpolicies
-    verbs:
-      - watch
-      - list
-  # Calico monitors its CRDs
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - globalfelixconfigs
-      - felixconfigurations
-      - bgppeers
-      - globalbgpconfigs
-      - bgpconfigurations
-      - ippools
-      - ipamblocks
-      - globalnetworkpolicies
-      - globalnetworksets
-      - networkpolicies
-      - networksets
-      - clusterinformations
-      - hostendpoints
-      - blockaffinities
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - ippools
-      - felixconfigurations
-      - clusterinformations
-    verbs:
-      - create
-      - update
-  # Calico may perform IPAM allocations
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - blockaffinities
-      - ipamblocks
-      - ipamhandles
-    verbs:
-      - get
-      - list
-      - create
-      - update
-      - delete
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - ipamconfigs
-    verbs:
-      - get
-  # Watch block affinities for route aggregation
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - blockaffinities
-    verbs:
-      - watch

resources/calico/config.yaml

@@ -1,45 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: calico-config
-  namespace: kube-system
-data:
-  # Disable Typha for now.
-  typha_service_name: "none"
-  # Calico backend to use
-  calico_backend: "bird"
-  # Calico MTU
-  veth_mtu: "${network_mtu}"
-  # The CNI network configuration to install on each node.
-  cni_network_config: |-
-    {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-            "type": "calico-ipam"
-          },
-          "policy": {
-            "type": "k8s"
-          },
-          "kubernetes": {
-            "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        },
-        {
-          "type": "bandwidth",
-          "capabilities": {"bandwidth": true}
-        }
-      ]
-    }

resources/calico/crd-bgpconfigurations.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: bgpconfigurations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BGPConfiguration
-    plural: bgpconfigurations
-    singular: bgpconfiguration

resources/calico/crd-bgppeers.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: bgppeers.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BGPPeer
-    plural: bgppeers
-    singular: bgppeer

resources/calico/crd-blockaffinities.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: blockaffinities.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BlockAffinity
-    plural: blockaffinities
-    singular: blockaffinity

resources/calico/crd-clusterinformations.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: clusterinformations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: ClusterInformation
-    plural: clusterinformations
-    singular: clusterinformation

resources/calico/crd-felixconfigurations.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: felixconfigurations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: FelixConfiguration
-    plural: felixconfigurations
-    singular: felixconfiguration

resources/calico/crd-globalnetworkpolicies.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: globalnetworkpolicies.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalNetworkPolicy
-    plural: globalnetworkpolicies
-    singular: globalnetworkpolicy

resources/calico/crd-globalnetworksets.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: globalnetworksets.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalNetworkSet
-    plural: globalnetworksets
-    singular: globalnetworkset

resources/calico/crd-hostendpoints.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: hostendpoints.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: HostEndpoint
-    plural: hostendpoints
-    singular: hostendpoint

resources/calico/crd-ipamblocks.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamblocks.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMBlock
-    plural: ipamblocks
-    singular: ipamblock

resources/calico/crd-ipamconfigs.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMConfig
-    plural: ipamconfigs
-    singular: ipamconfig

resources/calico/crd-ipamhandles.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamhandles.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMHandle
-    plural: ipamhandles
-    singular: ipamhandle

resources/calico/crd-ippools.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ippools.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPPool
-    plural: ippools
-    singular: ippool

resources/calico/crd-networkpolicies.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: networkpolicies.crd.projectcalico.org
-spec:
-  scope: Namespaced
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: NetworkPolicy
-    plural: networkpolicies
-    singular: networkpolicy

resources/calico/crd-networksets.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: networksets.crd.projectcalico.org
-spec:
-  scope: Namespaced
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: NetworkSet
-    plural: networksets
-    singular: networkset

resources/calico/daemonset.yaml

@@ -1,191 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    k8s-app: calico-node
-spec:
-  selector:
-    matchLabels:
-      k8s-app: calico-node
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: calico-node
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      priorityClassName: system-node-critical
-      serviceAccountName: calico-node
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      initContainers:
-        # Install Calico CNI binaries and CNI network config file on nodes
-        - name: install-cni
-          image: ${calico_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            # Name of the CNI config file to create on each node.
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            # Set node name based on k8s nodeName
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Contents of the CNI config to create on each node.
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-            - name: CNI_NET_DIR
-              value: "/etc/kubernetes/cni/net.d"
-            - name: CNI_MTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            - name: SLEEP
-              value: "false"
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
-      containers:
-        - name: calico-node
-          image: ${calico_image}
-          env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Wait for datastore
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # Typha support: controlled by the ConfigMap.
-            - name: FELIX_TYPHAK8SSERVICENAME
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: typha_service_name
-            - name: FELIX_USAGEREPORTINGENABLED
-              value: "${enable_reporting}"
-            # Set node name based on k8s nodeName.
-            - name: NODENAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Calico network backend
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-            # Cluster type to identify the deployment type
-            - name: CLUSTER_TYPE
-              value: "k8s,bgp"
-            # Auto-detect the BGP IP address.
-            - name: IP
-              value: "autodetect"
-            - name: IP_AUTODETECTION_METHOD
-              value: "${network_ip_autodetection_method}"
-            # Whether Felix should enable IP-in-IP tunnel
-            - name: FELIX_IPINIPENABLED
-              value: "${ipip_enabled}"
-            # MTU to set on the IPIP tunnel (if enabled)
-            - name: FELIX_IPINIPMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Whether Felix should enable VXLAN tunnel
-            - name: FELIX_VXLANENABLED
-              value: "${vxlan_enabled}"
-            # MTU to set on the VXLAN tunnel (if enabled)
-            - name: FELIX_VXLANMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            - name: NO_DEFAULT_POOLS
-              value: "true"
-            # Disable file logging so `kubectl logs` works.
-            - name: CALICO_DISABLE_FILE_LOGGING
-              value: "true"
-            # Set Felix endpoint to host default action to ACCEPT.
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
-            # Disable IPV6 on Kubernetes.
-            - name: FELIX_IPV6SUPPORT
-              value: "false"
-            # Enable felix info logging.
-            - name: FELIX_LOGSEVERITYSCREEN
-              value: "info"
-            - name: FELIX_HEALTHENABLED
-              value: "true"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 150m
-          livenessProbe:
-            exec:
-              command:
-                - /bin/calico-node
-                - -felix-ready
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-          readinessProbe:
-            exec:
-              command:
-                - /bin/calico-node
-                - -felix-ready
-                ${ipip_readiness}
-            periodSeconds: 10
-          volumeMounts:
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: var-lib-calico
-              mountPath: /var/lib/calico
-              readOnly: false
-            - name: var-run-calico
-              mountPath: /var/run/calico
-              readOnly: false
-            - name: xtables-lock
-              mountPath: /run/xtables.lock
-              readOnly: false
-      terminationGracePeriodSeconds: 0
-      volumes:
-        # Used by calico/node
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-lib-calico
-          hostPath:
-            path: /var/lib/calico
-        - name: var-run-calico
-          hostPath:
-            path: /var/run/calico
-        - name: xtables-lock
-          hostPath:
-            type: FileOrCreate
-            path: /run/xtables.lock
-        # Used by install-cni
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-conf-dir
-          hostPath:
-            path: /etc/kubernetes/cni/net.d

resources/calico/ippools-default-ipv4.yaml

@@ -1,10 +0,0 @@
-apiVersion: crd.projectcalico.org/v1
-kind: IPPool
-metadata:
-  name: default-ipv4-ippool
-spec:
-  blockSize: 24
-  cidr: ${pod_cidr}
-  ${network_encapsulation}
-  natOutgoing: true
-  nodeSelector: all()

resources/calico/service-account.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: calico-node
-  namespace: kube-system

resources/cilium/cluster-role-binding.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-agent
+subjects:
+- kind: ServiceAccount
+  name: cilium-agent
+  namespace: kube-system
+

resources/cilium/cluster-role.yaml

@@ -0,0 +1,188 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  # to automatically delete [core|kube]dns pods so that are starting to being
+  # managed by Cilium
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  # to perform LB IP allocation for BGP
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+  - services
+  - endpoints
+  # to check apiserver connectivity
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumendpoints/finalizers
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumnodes/finalizers
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
+  - ciliumendpointslices
+  - ciliumloadbalancerippools
+  - ciliumloadbalancerippools/status
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
+# Cilium leader elects if among multiple operator replicas
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-agent
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - pods
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumegressnatpolicies
+  - ciliumendpointslices
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+

resources/cilium/config.yaml

@@ -0,0 +1,175 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium
+  namespace: kube-system
+data:
+  # Identity allocation mode selects how identities are shared between cilium
+  # nodes by setting how they are stored. The options are "crd" or "kvstore".
+  # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+  #   These can be queried with:
+  #     kubectl get ciliumid
+  # - "kvstore" stores identities in a kvstore, etcd or consul, that is
+  #   configured below. Cilium versions before 1.6 supported only the kvstore
+  #   backend. Upgrades from these older cilium versions should continue using
+  #   the kvstore by commenting out the identity-allocation-mode below, or
+  #   setting it to "kvstore".
+  identity-allocation-mode: crd
+  cilium-endpoint-gc-interval: "5m0s"
+  nodes-gc-interval: "5m0s"
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "false"
+  # The agent can be put into the following three policy enforcement modes
+  # default, always and never.
+  # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes
+  enable-policy: "default"
+
+  # Prometheus
+  # enable-metrics: "true"
+  # prometheus-serve-addr: ":foo"
+  # operator-prometheus-serve-addr: ":bar"
+
+  # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
+  # address.
+  enable-ipv4: "true"
+
+  # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
+  # address.
+  enable-ipv6: "false"
+
+  # Enable probing for a more efficient clock source for the BPF datapath
+  enable-bpf-clock-probe: "true"
+
+  # Enable use of transparent proxying mechanisms (Linux 5.7+)
+  enable-bpf-tproxy: "false"
+
+  # If you want cilium monitor to aggregate tracing for packets, set this level
+  # to "low", "medium", or "maximum". The higher the level, the less packets
+  # that will be seen in monitor output.
+  monitor-aggregation: medium
+
+  # The monitor aggregation interval governs the typical time between monitor
+  # notification events for each allowed connection.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-interval: 5s
+
+  # The monitor aggregation flags determine which TCP flags which, upon the
+  # first observation, cause monitor notifications to be generated.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-flags: all
+
+  # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+  # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+  bpf-map-dynamic-size-ratio: "0.0025"
+  # bpf-policy-map-max specified the maximum number of entries in endpoint
+  # policy map (per endpoint)
+  bpf-policy-map-max: "16384"
+  # bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
+  # backend and affinity maps.
+  bpf-lb-map-max: "65536"
+
+  # Pre-allocation of map entries allows per-packet latency to be reduced, at
+  # the expense of up-front memory allocation for the entries in the maps. The
+  # default value below will minimize memory usage in the default installation;
+  # users who are sensitive to latency may consider setting this to "true".
+  #
+  # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+  # this option and behave as though it is set to "true".
+  #
+  # If this value is modified, then during the next Cilium startup the restore
+  # of existing endpoints and tracking of ongoing connections may be disrupted.
+  # As a result, reply packets may be dropped and the load-balancing decisions
+  # for established connections may change.
+  #
+  # If this option is set to "false" during an upgrade from 1.3 or earlier to
+  # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+  preallocate-bpf-maps: "false"
+
+  # Name of the cluster. Only relevant when building a mesh of clusters.
+  cluster-name: default
+  # Unique ID of the cluster. Must be unique across all conneted clusters and
+  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+  cluster-id: "0"
+
+  # Encapsulation mode for communication between nodes
+  # Possible values:
+  #   - disabled
+  #   - vxlan (default)
+  #   - geneve
+  routing-mode: "tunnel"
+  tunnel: vxlan
+  # Enables L7 proxy for L7 policy enforcement and visibility
+  enable-l7-proxy: "true"
+
+  auto-direct-node-routes: "false"
+
+  # enableXTSocketFallback enables the fallback compatibility solution
+  # when the xt_socket kernel module is missing and it is needed for
+  # the datapath L7 redirection to work properly.  See documentation
+  # for details on when this can be disabled:
+  # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version.
+  enable-xt-socket-fallback: "true"
+
+  # installIptablesRules enables installation of iptables rules to allow for
+  # TPROXY (L7 proxy injection), itpables based masquerading and compatibility
+  # with kube-proxy. See documentation for details on when this can be
+  # disabled.
+  install-iptables-rules: "true"
+
+  # masquerade traffic leaving the node destined for outside
+  enable-ipv4-masquerade: "true"
+  enable-ipv6-masquerade: "false"
+
+  # bpfMasquerade enables masquerading with BPF instead of iptables
+  enable-bpf-masquerade: "true"
+
+  # kube-proxy
+  kube-proxy-replacement:  "true"
+  kube-proxy-replacement-healthz-bind-address: ":10256"
+  enable-session-affinity: "true"
+
+  # ClusterIPs from host namespace
+  bpf-lb-sock: "true"
+  # ClusterIPs from external nodes
+  bpf-lb-external-clusterip: "true"
+
+  # NodePort
+  enable-node-port: "true"
+  enable-health-check-nodeport: "false"
+
+  # ExternalIPs
+  enable-external-ips: "true"
+
+  # HostPort
+  enable-host-port: "true"
+
+  # IPAM
+  ipam: "cluster-pool"
+  disable-cnp-status-updates: "true"
+  cluster-pool-ipv4-cidr: "${pod_cidr}"
+  cluster-pool-ipv4-mask-size: "24"
+
+  # Health
+  agent-health-port: "9876"
+  enable-health-checking: "true"
+  enable-endpoint-health-checking: "true"
+
+  # Identity
+  enable-well-known-identities: "false"
+  enable-remote-node-identity: "true"
+
+  # Misc
+  enable-bandwidth-manager: "false"
+  enable-local-redirect-policy: "false"
+  policy-audit-mode: "false"
+  operator-api-serve-addr: "127.0.0.1:9234"
+  enable-l2-neigh-discovery: "true"
+  enable-k8s-terminating-endpoint: "true"
+  enable-k8s-networkpolicy: "true"
+  external-envoy-proxy: "false"
+  write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+  cni-exclusive: "true"
+  cni-log-file: "/var/run/cilium/cilium-cni.log"

resources/cilium/daemonset.yaml

@@ -0,0 +1,219 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cilium
+  namespace: kube-system
+  labels:
+    k8s-app: cilium
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium-agent
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: cilium-agent
+    spec:
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      serviceAccountName: cilium-agent
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+      # Cilium v1.13.1 starts installing CNI plugins in yet another init container
+      # https://github.com/cilium/cilium/pull/24075
+      - name: install-cni
+        image: ${cilium_agent_image}
+        command:
+          - /install-plugin.sh
+        securityContext:
+          privileged: true
+          capabilities:
+            drop:
+              - ALL
+        volumeMounts:
+          - name: cni-bin-dir
+            mountPath: /host/opt/cni/bin
+
+      # Required to mount cgroup2 filesystem on the underlying Kubernetes node.
+      # We use nsenter command with host's cgroup and mount namespaces enabled.
+      - name: mount-cgroup
+        image: ${cilium_agent_image}
+        command:
+          - sh
+          - -ec
+          # The statically linked Go program binary is invoked to avoid any
+          # dependency on utilities like sh and mount that can be missing on certain
+          # distros installed on the underlying host. Copy the binary to the
+          # same directory where we install cilium cni plugin so that exec permissions
+          # are available.
+          - 'cp /usr/bin/cilium-mount /hostbin/cilium-mount && nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "$${BIN_PATH}/cilium-mount" $CGROUP_ROOT; rm /hostbin/cilium-mount'
+        env:
+          - name: CGROUP_ROOT
+            value: /run/cilium/cgroupv2
+          - name: BIN_PATH
+            value: /opt/cni/bin
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: hostproc
+            mountPath: /hostproc
+          - name: cni-bin-dir
+            mountPath: /hostbin
+
+      - name: clean-cilium-state
+        image: ${cilium_agent_image}
+        command:
+        - /init-container.sh
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        # Required to mount cgroup filesystem from the host to cilium agent pod
+        - name: cilium-cgroup
+          mountPath: /run/cilium/cgroupv2
+          mountPropagation: HostToContainer
+
+      containers:
+      - name: cilium-agent
+        image: ${cilium_agent_image}
+        command:
+        - cilium-agent
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        ports:
+          # Not yet used, prefer exec's
+          - name: health
+            protocol: TCP
+            containerPort: 9876
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        securityContext:
+          privileged: true
+        livenessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 30
+          initialDelaySeconds: 120
+          successThreshold: 1
+          failureThreshold: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 20
+          initialDelaySeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+          timeoutSeconds: 5
+        volumeMounts:
+        # Load kernel modules
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+        # Keep state between restarts
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
+        # Configuration
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+        # Install config on host
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      terminationGracePeriodSeconds: 1
+      volumes:
+      # Load kernel modules
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      # Access iptables concurrently with other processes (e.g. kube-proxy)
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock
+      # Keep state between restarts
+      - name: var-run-cilium
+        hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+      # Keep state between restarts for bpf maps
+      - name: sys-fs-bpf
+        hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+      # Mount host cgroup2 filesystem
+      - name: hostproc
+        hostPath:
+          path: /proc
+          type: Directory
+      - name: cilium-cgroup
+        hostPath:
+          path: /run/cilium/cgroupv2
+          type: DirectoryOrCreate
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium
+      # Install CNI plugin and config on host
+      - name: cni-bin-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path:  /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path: /etc/cni/net.d
+

resources/cilium/deployment.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      name: cilium-operator
+  template:
+    metadata:
+      labels:
+        name: cilium-operator
+    spec:
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      serviceAccountName: cilium-operator
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      containers:
+      - name: cilium-operator
+        image: ${cilium_operator_image}
+        command:
+        - cilium-operator-generic
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        - --debug=$(CILIUM_DEBUG)
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        - name: CILIUM_DEBUG
+          valueFrom:
+            configMapKeyRef:
+              name: cilium
+              key: debug
+              optional: true
+        ports:
+          - name: health
+            protocol: TCP
+            containerPort: 9234
+        livenessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 3
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          periodSeconds: 15
+          timeoutSeconds: 3
+          failureThreshold: 5
+        volumeMounts:
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+      topologySpreadConstraints:
+        - topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              name: cilium-operator
+          maxSkew: 1
+          whenUnsatisfiable: DoNotSchedule
+      volumes:
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium

resources/cilium/service-account.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-agent
+  namespace: kube-system
+

resources/coredns/cluster-role.yaml

@@ -14,6 +14,12 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups: ["discovery.k8s.io"]
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
   - apiGroups: [""]
     resources:
       - nodes

resources/coredns/deployment.yaml

@@ -21,10 +21,15 @@ spec:
       labels:
         tier: control-plane
         k8s-app: coredns
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node.kubernetes.io/controller
+                operator: Exists
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
@@ -41,9 +46,12 @@ spec:
                   - coredns
               topologyKey: kubernetes.io/hostname
       priorityClassName: system-cluster-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: coredns
       tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node-role.kubernetes.io/controller
           effect: NoSchedule
       containers:
         - name: coredns

resources/flannel/config.yaml

@@ -32,6 +32,6 @@ data:
       "Network": "${pod_cidr}",
       "Backend": {
         "Type": "vxlan",
-        "Port": 4789
+        "Port": 8472
       }
     }

resources/flannel/daemonset.yaml

@@ -17,17 +17,37 @@ spec:
     metadata:
       labels:
         k8s-app: flannel
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       hostNetwork: true
       priorityClassName: system-node-critical
       serviceAccountName: flannel
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+        - name: install-cni
+          image: ${flannel_cni_image}
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: flannel-config
+                  key: cni-conf.json
+          volumeMounts:
+            - name: cni-bin-dir
+              mountPath: /host/opt/cni/bin/
+            - name: cni-conf-dir
+              mountPath: /host/etc/cni/net.d
       containers:
         - name: flannel
           image: ${flannel_image}
@@ -55,20 +75,8 @@ spec:
               mountPath: /etc/kube-flannel/
             - name: run-flannel
               mountPath: /run/flannel
-        - name: install-cni
-          image: ${flannel_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: flannel-config
-                  key: cni-conf.json
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin/
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
+            - name: xtables-lock
+              mountPath: /run/xtables.lock
       volumes:
         - name: flannel-config
           configMap:
@@ -82,4 +90,10 @@ spec:
             path: /opt/cni/bin
         - name: cni-conf-dir
           hostPath:
-            path: /etc/kubernetes/cni/net.d
+            type: DirectoryOrCreate
+            path: /etc/cni/net.d
+        # Access iptables concurrently
+        - name: xtables-lock
+          hostPath:
+            type: FileOrCreate
+            path: /run/xtables.lock

resources/kube-proxy/kube-proxy.yaml

@@ -20,32 +20,42 @@ spec:
       labels:
         tier: node
         k8s-app: kube-proxy
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       hostNetwork: true
       priorityClassName: system-node-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: kube-proxy
       tolerations:
-      - effect: NoSchedule
+      - key: node-role.kubernetes.io/controller
         operator: Exists
-      - effect: NoExecute
+      - key: node.kubernetes.io/not-ready
         operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       containers:
       - name: kube-proxy
-        image: ${hyperkube_image}
+        image: ${kube_proxy_image}
         command:
-        - ./hyperkube
         - kube-proxy
         - --cluster-cidr=${pod_cidr}
         - --hostname-override=$(NODE_NAME)
         - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --metrics-bind-address=0.0.0.0
         - --proxy-mode=ipvs
         env:
           - name: NODE_NAME
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
+        ports:
+        - name: metrics
+          containerPort: 10249
+        - name: health
+          containerPort: 10256
         livenessProbe:
           httpGet:
             path: /healthz
@@ -61,9 +71,14 @@ spec:
         - name: lib-modules
           mountPath: /lib/modules
           readOnly: true
-        - name: ssl-certs-host
+        - name: etc-ssl
           mountPath: /etc/ssl/certs
           readOnly: true
+        - name: etc-pki
+          mountPath: /etc/pki
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
       volumes:
       - name: kubeconfig
         configMap:
@@ -71,6 +86,14 @@ spec:
       - name: lib-modules
         hostPath:
           path: /lib/modules
-      - name: ssl-certs-host
+      - name: etc-ssl
         hostPath:
-          path: ${trusted_certs_dir}
+          path: /etc/ssl/certs
+      - name: etc-pki
+        hostPath:
+          path: /etc/pki
+      # Access iptables concurrently
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock

resources/kube-router/cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-router
-subjects:
-  - kind: ServiceAccount
-    name: kube-router
-    namespace: kube-system

resources/kube-router/cluster-role.yaml

@@ -1,33 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: kube-router
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces
-      - pods
-      - services
-      - nodes
-      - endpoints
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-      - "networking.k8s.io"
-    resources:
-      - networkpolicies
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-      - extensions
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch

resources/kube-router/config.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-config
-  namespace: kube-system
-data:
-  cni-conf.json: |
-    {
-      "name": "pod-network",
-      "cniVersion": "0.3.1",
-      "plugins":[
-        {
-          "name": "kube-router",
-          "type": "bridge",
-          "bridge": "kube-bridge",
-          "isDefaultGateway": true,
-          "mtu": ${network_mtu},
-          "ipam": {
-            "type": "host-local"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {
-            "portMappings": true
-          }
-        }
-      ]
-    }

resources/kube-router/daemonset.yaml

@@ -1,90 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-router
-  namespace: kube-system
-  labels:
-    k8s-app: kube-router
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-router
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-router
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      priorityClassName: system-node-critical
-      serviceAccountName: kube-router
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      containers:
-        - name: kube-router
-          image: ${kube_router_image}
-          args:
-            - --kubeconfig=/etc/kubernetes/kubeconfig
-            - --run-router=true
-            - --run-firewall=true
-            - --run-service-proxy=false
-            - --v=5
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: KUBE_ROUTER_CNI_CONF_FILE
-              value: /etc/cni/net.d/10-kuberouter.conflist
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: cni-conf-dir
-              mountPath: /etc/cni/net.d
-            - name: kubeconfig
-              mountPath: /etc/kubernetes
-              readOnly: true
-        - name: install-cni
-          image: ${flannel_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_OLD_NAME
-              value: 10-flannel.conflist
-            - name: CNI_CONF_NAME
-              value: 10-kuberouter.conflist
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: kube-router-config
-                  key: cni-conf.json
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
-      volumes:
-        # Used by kube-router
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: kubeconfig
-          configMap:
-            name: kubeconfig-in-cluster
-        # Used by install-cni
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-conf-dir
-          hostPath:
-            path: /etc/kubernetes/cni/net.d

resources/kube-router/service-account.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-router
-  namespace: kube-system

resources/kubeconfig-admin

@@ -1,18 +1,18 @@
 apiVersion: v1
 kind: Config
 clusters:
-- name: ${name}-cluster
+- name: ${name}
   cluster:
     server: ${server}
     certificate-authority-data: ${ca_cert}
 users:
-- name: ${name}-user
+- name: ${name}
   user:
     client-certificate-data: ${kubelet_cert}
     client-key-data: ${kubelet_key}
-current-context: ${name}-context
+current-context: ${name}
 contexts:
-- name: ${name}-context
+- name: ${name}
   context:
-    cluster: ${name}-cluster
-    user: ${name}-user
+    cluster: ${name}
+    user: ${name}

resources/kubeconfig-bootstrap

@@ -8,8 +8,7 @@ clusters:
 users:
 - name: kubelet
   user:
-    client-certificate-data: ${kubelet_cert}
-    client-key-data: ${kubelet_key}
+    token: ${token_id}.${token_secret}
 contexts:
 - context:
     cluster: local

resources/manifests/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Bind system:bootstrappers to ClusterRole for node bootstrap
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-bootstrapper
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve new CSRs from "system:bootstrappers" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-new
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve renewal CSRs from "system:nodes" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-renew
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/bootstrap-token.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>"
+  name: bootstrap-token-${token_id}
+  namespace: kube-system
+stringData:
+  description: "Typhoon generated bootstrap token"
+  token-id: ${token_id}
+  token-secret: ${token_secret}
+  usage-bootstrap-authentication: "true"

resources/manifests/in-cluster.yaml

@@ -0,0 +1,10 @@
+# in-cluster ConfigMap is for control plane components that must reach
+# kube-apiserver before service IPs are available (e.g. 10.3.0.1)
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: in-cluster
+  namespace: kube-system
+data:
+  apiserver-host: ${apiserver_host}
+  apiserver-port: "${apiserver_port}"

resources/manifests/kubelet-delete-cluster-role-binding.yaml

@@ -7,6 +7,6 @@ roleRef:
   kind: ClusterRole
   name: kubelet-delete
 subjects:
-- kind: Group
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
   name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/manifests/kubelet-delete-cluster-role.yaml

@@ -8,3 +8,16 @@ rules:
       - nodes
     verbs:
       - delete
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+    verbs:
+      - get
+      - list
+  - apiGroups: [""]
+    resources:
+      - pods/eviction
+    verbs:
+      - create

resources/manifests/kubelet-nodes-cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system-nodes
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node
-subjects:
-- kind: Group
-  name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/static-manifests/kube-apiserver.yaml

@@ -6,39 +6,43 @@ metadata:
   labels:
     k8s-app: kube-apiserver
     tier: control-plane
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
   hostNetwork: true
   priorityClassName: system-cluster-critical
   securityContext:
     runAsNonRoot: true
     runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
   containers:
   - name: kube-apiserver
-    image: ${hyperkube_image}
+    image: ${kube_apiserver_image}
     command:
-    - /hyperkube
     - kube-apiserver
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --anonymous-auth=false
-    - --authorization-mode=RBAC
-    - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --cloud-provider=${cloud_provider}
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
+    - --authorization-mode=Node,RBAC
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
+    - --enable-admission-plugins=NodeRestriction
+    - --enable-bootstrap-token-auth=true
+    - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.crt
+    - --etcd-certfile=/etc/kubernetes/pki/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/pki/etcd-client.key
     - --etcd-servers=${etcd_servers}
-    - --insecure-port=0
-    - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-    - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
+    - --feature-gates=MutatingAdmissionPolicy=true
+    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.crt
+    - --kubelet-client-key=/etc/kubernetes/pki/apiserver.key
     - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
+    - --runtime-config=admissionregistration.k8s.io/v1beta1=true,admissionregistration.k8s.io/v1alpha1=true
     - --secure-port=6443
-    - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
+    - --service-account-issuer=${service_account_issuer}
+    - --service-account-jwks-uri=${service_account_issuer}/openid/v1/jwks
+    - --service-account-key-file=/etc/kubernetes/pki/service-account.pub
+    - --service-account-signing-key-file=/etc/kubernetes/pki/service-account.key
     - --service-cluster-ip-range=${service_cidr}
-    - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-    - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
+    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
+    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
     env:
     - name: POD_IP
       valueFrom:
@@ -46,18 +50,24 @@ spec:
           fieldPath: status.podIP
     resources:
       requests:
-        cpu: 200m
+        cpu: 150m
     volumeMounts:
     - name: secrets
-      mountPath: /etc/kubernetes/secrets
+      mountPath: /etc/kubernetes/pki
       readOnly: true
-    - name: ssl-certs-host
+    - name: etc-ssl
       mountPath: /etc/ssl/certs
       readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
   volumes:
   - name: secrets
     hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
-  - name: ssl-certs-host
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
     hostPath:
-      path: ${trusted_certs_dir}
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki

resources/static-manifests/kube-controller-manager.yaml

@@ -6,53 +6,70 @@ metadata:
   labels:
     k8s-app: kube-controller-manager
     tier: control-plane
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
   hostNetwork: true
   priorityClassName: system-cluster-critical
   securityContext:
     runAsNonRoot: true
     runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
   containers:
   - name: kube-controller-manager
-    image: ${hyperkube_image}
+    image: ${kube_controller_manager_image}
     command:
-    - /hyperkube
     - kube-controller-manager
+    - --authentication-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
     - --allocate-node-cidrs=true
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
     - --cluster-cidr=${pod_cidr}
-    - --service-cluster-ip-range=${service_cidr}
-    - --cloud-provider=${cloud_provider}
-    - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
-    - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
+    - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
+    - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
+    - --cluster-signing-duration=72h
+    - --controllers=*,tokencleaner
     - --configure-cloud-routes=false
-    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
+    - --kubeconfig=/etc/kubernetes/pki/controller-manager.conf
     - --leader-elect=true
-    - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
+    - --root-ca-file=/etc/kubernetes/pki/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/pki/service-account.key
+    - --service-cluster-ip-range=${service_cidr}
+    - --use-service-account-credentials=true
     livenessProbe:
       httpGet:
         scheme: HTTPS
         host: 127.0.0.1
         path: /healthz
         port: 10257
-      initialDelaySeconds: 15
+      initialDelaySeconds: 25
       timeoutSeconds: 15
+      failureThreshold: 8
     resources:
       requests:
-        cpu: 200m
+        cpu: 150m
     volumeMounts:
     - name: secrets
-      mountPath: /etc/kubernetes/secrets
+      mountPath: /etc/kubernetes/pki
       readOnly: true
-    - name: ssl-host
+    - name: etc-ssl
       mountPath: /etc/ssl/certs
       readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
+    - name: flex
+      mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
   volumes:
   - name: secrets
     hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
-  - name: ssl-host
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
     hostPath:
-      path: ${trusted_certs_dir}
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki
+  - name: flex
+    hostPath:
+      type: DirectoryOrCreate
+      path: /var/lib/kubelet/volumeplugins

resources/static-manifests/kube-scheduler.yaml

@@ -6,21 +6,22 @@ metadata:
   labels:
     k8s-app: kube-scheduler
     tier: control-plane
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
   hostNetwork: true
   priorityClassName: system-cluster-critical
   securityContext:
     runAsNonRoot: true
     runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
   containers:
   - name: kube-scheduler
-    image: ${hyperkube_image}
+    image: ${kube_scheduler_image}
     command:
-    - /hyperkube
     - kube-scheduler
-    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
+    - --authentication-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --kubeconfig=/etc/kubernetes/pki/scheduler.conf
     - --leader-elect=true
     livenessProbe:
       httpGet:
@@ -35,9 +36,9 @@ spec:
         cpu: 100m
     volumeMounts:
     - name: secrets
-      mountPath: /etc/kubernetes/secrets
+      mountPath: /etc/kubernetes/pki/scheduler.conf
       readOnly: true
   volumes:
   - name: secrets
     hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
+      path: /etc/kubernetes/pki/scheduler.conf

terraform.tfvars.example

@@ -1,5 +1,4 @@
 cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
-asset_dir = "/home/core/mycluster"
 networking = "flannel"

tls-aggregation.tf

@@ -1,3 +1,12 @@
+locals {
+  # Kubernetes Aggregation TLS assets map
+  aggregation_tls = var.enable_aggregation ? {
+    "tls/k8s/aggregation-ca.crt"     = tls_self_signed_cert.aggregation-ca[0].cert_pem,
+    "tls/k8s/aggregation-client.crt" = tls_locally_signed_cert.aggregation-client[0].cert_pem,
+    "tls/k8s/aggregation-client.key" = tls_private_key.aggregation-client[0].private_key_pem,
+  } : {}
+}
+
 # Kubernetes Aggregation CA (i.e. front-proxy-ca)
 # Files: tls/{aggregation-ca.crt,aggregation-ca.key}
 
@@ -11,7 +20,6 @@ resource "tls_private_key" "aggregation-ca" {
 resource "tls_self_signed_cert" "aggregation-ca" {
   count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = tls_private_key.aggregation-ca[0].algorithm
   private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
 
   subject {
@@ -28,20 +36,6 @@ resource "tls_self_signed_cert" "aggregation-ca" {
   ]
 }
 
-resource "local_file" "aggregation-ca-key" {
-  count = var.enable_aggregation ? 1 : 0
-
-  content  = tls_private_key.aggregation-ca[0].private_key_pem
-  filename = "${var.asset_dir}/tls/aggregation-ca.key"
-}
-
-resource "local_file" "aggregation-ca-crt" {
-  count = var.enable_aggregation ? 1 : 0
-
-  content  = tls_self_signed_cert.aggregation-ca[0].cert_pem
-  filename = "${var.asset_dir}/tls/aggregation-ca.crt"
-}
-
 # Kubernetes apiserver (i.e. front-proxy-client)
 # Files: tls/{aggregation-client.crt,aggregation-client.key}
 
@@ -55,7 +49,6 @@ resource "tls_private_key" "aggregation-client" {
 resource "tls_cert_request" "aggregation-client" {
   count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = tls_private_key.aggregation-client[0].algorithm
   private_key_pem = tls_private_key.aggregation-client[0].private_key_pem
 
   subject {
@@ -68,7 +61,6 @@ resource "tls_locally_signed_cert" "aggregation-client" {
 
   cert_request_pem = tls_cert_request.aggregation-client[0].cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.aggregation-ca[0].key_algorithm
   ca_private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
   ca_cert_pem        = tls_self_signed_cert.aggregation-ca[0].cert_pem
 
@@ -81,17 +73,3 @@ resource "tls_locally_signed_cert" "aggregation-client" {
   ]
 }
 
-resource "local_file" "aggregation-client-key" {
-  count = var.enable_aggregation ? 1 : 0
-
-  content  = tls_private_key.aggregation-client[0].private_key_pem
-  filename = "${var.asset_dir}/tls/aggregation-client.key"
-}
-
-resource "local_file" "aggregation-client-crt" {
-  count = var.enable_aggregation ? 1 : 0
-
-  content  = tls_locally_signed_cert.aggregation-client[0].cert_pem
-  filename = "${var.asset_dir}/tls/aggregation-client.crt"
-}
-

tls-etcd.tf

@@ -1,70 +1,19 @@
-# etcd-ca.crt
-resource "local_file" "etcd_ca_crt" {
-  content  = tls_self_signed_cert.etcd-ca.cert_pem
-  filename = "${var.asset_dir}/tls/etcd-ca.crt"
+locals {
+  # etcd TLS assets map
+  etcd_tls = {
+    "tls/etcd/etcd-client-ca.crt" = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/etcd-client.crt"    = tls_locally_signed_cert.client.cert_pem,
+    "tls/etcd/etcd-client.key"    = tls_private_key.client.private_key_pem
+    "tls/etcd/server-ca.crt"      = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/server.crt"         = tls_locally_signed_cert.server.cert_pem
+    "tls/etcd/server.key"         = tls_private_key.server.private_key_pem
+    "tls/etcd/peer-ca.crt"        = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/peer.crt"           = tls_locally_signed_cert.peer.cert_pem
+    "tls/etcd/peer.key"           = tls_private_key.peer.private_key_pem
+  }
 }
 
-# etcd-ca.key
-resource "local_file" "etcd_ca_key" {
-  content  = tls_private_key.etcd-ca.private_key_pem
-  filename = "${var.asset_dir}/tls/etcd-ca.key"
-}
-
-# etcd-client-ca.crt
-resource "local_file" "etcd_client_ca_crt" {
-  content  = tls_self_signed_cert.etcd-ca.cert_pem
-  filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
-}
-
-# etcd-client.crt
-resource "local_file" "etcd_client_crt" {
-  content  = tls_locally_signed_cert.client.cert_pem
-  filename = "${var.asset_dir}/tls/etcd-client.crt"
-}
-
-# etcd-client.key
-resource "local_file" "etcd_client_key" {
-  content  = tls_private_key.client.private_key_pem
-  filename = "${var.asset_dir}/tls/etcd-client.key"
-}
-
-# server-ca.crt
-resource "local_file" "etcd_server_ca_crt" {
-  content  = tls_self_signed_cert.etcd-ca.cert_pem
-  filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
-}
-
-# server.crt
-resource "local_file" "etcd_server_crt" {
-  content  = tls_locally_signed_cert.server.cert_pem
-  filename = "${var.asset_dir}/tls/etcd/server.crt"
-}
-
-# server.key
-resource "local_file" "etcd_server_key" {
-  content  = tls_private_key.server.private_key_pem
-  filename = "${var.asset_dir}/tls/etcd/server.key"
-}
-
-# peer-ca.crt
-resource "local_file" "etcd_peer_ca_crt" {
-  content  = tls_self_signed_cert.etcd-ca.cert_pem
-  filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
-}
-
-# peer.crt
-resource "local_file" "etcd_peer_crt" {
-  content  = tls_locally_signed_cert.peer.cert_pem
-  filename = "${var.asset_dir}/tls/etcd/peer.crt"
-}
-
-# peer.key
-resource "local_file" "etcd_peer_key" {
-  content  = tls_private_key.peer.private_key_pem
-  filename = "${var.asset_dir}/tls/etcd/peer.key"
-}
-
-# certificates and keys
+# etcd CA
 
 resource "tls_private_key" "etcd-ca" {
   algorithm = "RSA"
@@ -72,7 +21,6 @@ resource "tls_private_key" "etcd-ca" {
 }
 
 resource "tls_self_signed_cert" "etcd-ca" {
-  key_algorithm   = tls_private_key.etcd-ca.algorithm
   private_key_pem = tls_private_key.etcd-ca.private_key_pem
 
   subject {
@@ -90,15 +38,14 @@ resource "tls_self_signed_cert" "etcd-ca" {
   ]
 }
 
-# client certs are used for client (apiserver, locksmith, etcd-operator)
-# to etcd communication
+# etcd Client (apiserver to etcd communication)
+
 resource "tls_private_key" "client" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "client" {
-  key_algorithm   = tls_private_key.client.algorithm
   private_key_pem = tls_private_key.client.private_key_pem
 
   subject {
@@ -116,7 +63,6 @@ resource "tls_cert_request" "client" {
 resource "tls_locally_signed_cert" "client" {
   cert_request_pem = tls_cert_request.client.cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
   ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
   ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
@@ -130,13 +76,14 @@ resource "tls_locally_signed_cert" "client" {
   ]
 }
 
+# etcd Server
+
 resource "tls_private_key" "server" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "server" {
-  key_algorithm   = tls_private_key.server.algorithm
   private_key_pem = tls_private_key.server.private_key_pem
 
   subject {
@@ -154,7 +101,6 @@ resource "tls_cert_request" "server" {
 resource "tls_locally_signed_cert" "server" {
   cert_request_pem = tls_cert_request.server.cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
   ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
   ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
@@ -168,13 +114,14 @@ resource "tls_locally_signed_cert" "server" {
   ]
 }
 
+# etcd Peer
+
 resource "tls_private_key" "peer" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "peer" {
-  key_algorithm   = tls_private_key.peer.algorithm
   private_key_pem = tls_private_key.peer.private_key_pem
 
   subject {
@@ -188,7 +135,6 @@ resource "tls_cert_request" "peer" {
 resource "tls_locally_signed_cert" "peer" {
   cert_request_pem = tls_cert_request.peer.cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
   ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
   ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 

tls-k8s.tf

@@ -1,3 +1,15 @@
+locals {
+  # Kubernetes TLS assets map
+  kubernetes_tls = {
+    "tls/k8s/ca.crt"              = tls_self_signed_cert.kube-ca.cert_pem,
+    "tls/k8s/ca.key"              = tls_private_key.kube-ca.private_key_pem,
+    "tls/k8s/apiserver.crt"       = tls_locally_signed_cert.apiserver.cert_pem,
+    "tls/k8s/apiserver.key"       = tls_private_key.apiserver.private_key_pem,
+    "tls/k8s/service-account.pub" = tls_private_key.service-account.public_key_pem
+    "tls/k8s/service-account.key" = tls_private_key.service-account.private_key_pem
+  }
+}
+
 # Kubernetes CA (tls/{ca.crt,ca.key})
 
 resource "tls_private_key" "kube-ca" {
@@ -6,7 +18,6 @@ resource "tls_private_key" "kube-ca" {
 }
 
 resource "tls_self_signed_cert" "kube-ca" {
-  key_algorithm   = tls_private_key.kube-ca.algorithm
   private_key_pem = tls_private_key.kube-ca.private_key_pem
 
   subject {
@@ -24,16 +35,6 @@ resource "tls_self_signed_cert" "kube-ca" {
   ]
 }
 
-resource "local_file" "kube-ca-key" {
-  content  = tls_private_key.kube-ca.private_key_pem
-  filename = "${var.asset_dir}/tls/ca.key"
-}
-
-resource "local_file" "kube-ca-crt" {
-  content  = tls_self_signed_cert.kube-ca.cert_pem
-  filename = "${var.asset_dir}/tls/ca.crt"
-}
-
 # Kubernetes API Server (tls/{apiserver.key,apiserver.crt})
 
 resource "tls_private_key" "apiserver" {
@@ -42,7 +43,6 @@ resource "tls_private_key" "apiserver" {
 }
 
 resource "tls_cert_request" "apiserver" {
-  key_algorithm   = tls_private_key.apiserver.algorithm
   private_key_pem = tls_private_key.apiserver.private_key_pem
 
   subject {
@@ -66,7 +66,6 @@ resource "tls_cert_request" "apiserver" {
 resource "tls_locally_signed_cert" "apiserver" {
   cert_request_pem = tls_cert_request.apiserver.cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
   ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
   ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
@@ -80,14 +79,64 @@ resource "tls_locally_signed_cert" "apiserver" {
   ]
 }
 
-resource "local_file" "apiserver-key" {
-  content  = tls_private_key.apiserver.private_key_pem
-  filename = "${var.asset_dir}/tls/apiserver.key"
+# kube-controller-manager
+
+resource "tls_private_key" "controller-manager" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
 }
 
-resource "local_file" "apiserver-crt" {
-  content  = tls_locally_signed_cert.apiserver.cert_pem
-  filename = "${var.asset_dir}/tls/apiserver.crt"
+resource "tls_cert_request" "controller-manager" {
+  private_key_pem = tls_private_key.controller-manager.private_key_pem
+
+  subject {
+    common_name = "system:kube-controller-manager"
+  }
+}
+
+resource "tls_locally_signed_cert" "controller-manager" {
+  cert_request_pem = tls_cert_request.controller-manager.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+# kube-scheduler
+
+resource "tls_private_key" "scheduler" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "scheduler" {
+  private_key_pem = tls_private_key.scheduler.private_key_pem
+
+  subject {
+    common_name = "system:kube-scheduler"
+  }
+}
+
+resource "tls_locally_signed_cert" "scheduler" {
+  cert_request_pem = tls_cert_request.scheduler.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
 }
 
 # Kubernetes Admin (tls/{admin.key,admin.crt})
@@ -98,7 +147,6 @@ resource "tls_private_key" "admin" {
 }
 
 resource "tls_cert_request" "admin" {
-  key_algorithm   = tls_private_key.admin.algorithm
   private_key_pem = tls_private_key.admin.private_key_pem
 
   subject {
@@ -110,7 +158,6 @@ resource "tls_cert_request" "admin" {
 resource "tls_locally_signed_cert" "admin" {
   cert_request_pem = tls_cert_request.admin.cert_request_pem
 
-  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
   ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
   ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
@@ -123,16 +170,6 @@ resource "tls_locally_signed_cert" "admin" {
   ]
 }
 
-resource "local_file" "admin-key" {
-  content  = tls_private_key.admin.private_key_pem
-  filename = "${var.asset_dir}/tls/admin.key"
-}
-
-resource "local_file" "admin-crt" {
-  content  = tls_locally_signed_cert.admin.cert_pem
-  filename = "${var.asset_dir}/tls/admin.crt"
-}
-
 # Kubernete's Service Account (tls/{service-account.key,service-account.pub})
 
 resource "tls_private_key" "service-account" {
@@ -140,57 +177,3 @@ resource "tls_private_key" "service-account" {
   rsa_bits  = "2048"
 }
 
-resource "local_file" "service-account-key" {
-  content  = tls_private_key.service-account.private_key_pem
-  filename = "${var.asset_dir}/tls/service-account.key"
-}
-
-resource "local_file" "service-account-crt" {
-  content  = tls_private_key.service-account.public_key_pem
-  filename = "${var.asset_dir}/tls/service-account.pub"
-}
-
-# Kubelet
-
-resource "tls_private_key" "kubelet" {
-  algorithm = "RSA"
-  rsa_bits  = "2048"
-}
-
-resource "tls_cert_request" "kubelet" {
-  key_algorithm   = tls_private_key.kubelet.algorithm
-  private_key_pem = tls_private_key.kubelet.private_key_pem
-
-  subject {
-    common_name  = "kubelet"
-    organization = "system:nodes"
-  }
-}
-
-resource "tls_locally_signed_cert" "kubelet" {
-  cert_request_pem = tls_cert_request.kubelet.cert_request_pem
-
-  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
-  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
-  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
-
-  validity_period_hours = 8760
-
-  allowed_uses = [
-    "key_encipherment",
-    "digital_signature",
-    "server_auth",
-    "client_auth",
-  ]
-}
-
-resource "local_file" "kubelet-key" {
-  content  = tls_private_key.kubelet.private_key_pem
-  filename = "${var.asset_dir}/tls/kubelet.key"
-}
-
-resource "local_file" "kubelet-crt" {
-  content  = tls_locally_signed_cert.kubelet.cert_pem
-  filename = "${var.asset_dir}/tls/kubelet.crt"
-}
-

variables.tf

@@ -13,89 +13,59 @@ variable "etcd_servers" {
   description = "List of URLs used to reach etcd servers."
 }
 
-variable "asset_dir" {
-  type        = string
-  description = "Absolute path to a directory where generated assets should be placed (contains secrets)"
-}
-
-variable "cloud_provider" {
-  type        = string
-  description = "The provider for cloud services (empty string for no provider)"
-  default     = ""
-}
+# optional
 
 variable "networking" {
   type        = string
-  description = "Choice of networking provider (flannel or calico or kube-router)"
-  default     = "flannel"
-}
-
-variable "network_mtu" {
-  type        = number
-  description = "CNI interface MTU (only applies to calico and kube-router)"
-  default     = 1500
-}
-
-variable "network_encapsulation" {
-  type        = string
-  description = "Network encapsulation mode either ipip or vxlan (only applies to calico)"
-  default     = "ipip"
-}
-
-variable "network_ip_autodetection_method" {
-  type        = string
-  description = "Method to autodetect the host IPv4 address (only applies to calico)"
-  default     = "first-found"
+  description = "Choice of networking provider (flannel or cilium)"
+  default     = "cilium"
+  validation {
+    condition     = contains(["flannel", "cilium"], var.networking)
+    error_message = "networking can be flannel or cilium."
+  }
 }
 
 variable "pod_cidr" {
   type        = string
   description = "CIDR IP range to assign Kubernetes pods"
-  default     = "10.2.0.0/16"
+  default     = "10.20.0.0/14"
 }
 
 variable "service_cidr" {
-  type    = string
+  type        = string
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
 The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
 EOD
-  default = "10.3.0.0/24"
+  default     = "10.3.0.0/24"
 }
 
-
 variable "container_images" {
   type        = map(string)
   description = "Container images to use"
-
   default = {
-    calico      = "quay.io/calico/node:v3.10.1"
-    calico_cni  = "quay.io/calico/cni:v3.10.1"
-    flannel     = "quay.io/coreos/flannel:v0.11.0-amd64"
-    flannel_cni = "quay.io/coreos/flannel-cni:v0.3.0"
-    kube_router = "cloudnativelabs/kube-router:v0.3.2"
-    hyperkube   = "k8s.gcr.io/hyperkube:v1.16.3"
-    coredns     = "k8s.gcr.io/coredns:1.6.5"
+    cilium_agent            = "quay.io/cilium/cilium:v1.18.4"
+    cilium_operator         = "quay.io/cilium/operator-generic:v1.18.4"
+    coredns                 = "registry.k8s.io/coredns/coredns:v1.13.1"
+    flannel                 = "docker.io/flannel/flannel:v0.27.0"
+    flannel_cni             = "quay.io/poseidon/flannel-cni:v0.4.2"
+    kube_apiserver          = "registry.k8s.io/kube-apiserver:v1.34.2"
+    kube_controller_manager = "registry.k8s.io/kube-controller-manager:v1.34.2"
+    kube_scheduler          = "registry.k8s.io/kube-scheduler:v1.34.2"
+    kube_proxy              = "registry.k8s.io/kube-proxy:v1.34.2"
   }
 }
 
-
-variable "trusted_certs_dir" {
-  type        = string
-  description = "Path to the directory on cluster nodes where trust TLS certs are kept"
-  default     = "/usr/share/ca-certificates"
-}
-
-variable "enable_reporting" {
-  type        = bool
-  description = "Enable usage or analytics reporting to upstream component owners (Tigera: Calico)"
-  default     = false
-}
-
 variable "enable_aggregation" {
   type        = bool
-  description = "Enable the Kubernetes Aggregation Layer (defaults to false, recommended)"
-  default     = false
+  description = "Enable the Kubernetes Aggregation Layer (defaults to true)"
+  default     = true
+}
+
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
 }
 
 # unofficial, temporary, may be removed without notice
@@ -112,3 +82,59 @@ variable "cluster_domain_suffix" {
   default     = "cluster.local"
 }
 
+variable "components" {
+  description = "Configure pre-installed cluster components"
+  type = object({
+    enable = optional(bool, true)
+    coredns = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    kube_proxy = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    # CNI providers are enabled for pre-install by default, but only the
+    # provider matching var.networking is actually installed.
+    flannel = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    cilium = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+  })
+  default = {
+    enable     = true
+    coredns    = null
+    kube_proxy = null
+    flannel    = null
+    cilium     = null
+  }
+  # Set the variable value to the default value when the caller
+  # sets it to null.
+  nullable = false
+}
+
+variable "service_account_issuer" {
+  type        = string
+  description = "kube-apiserver service account token issuer (used as an identifier in 'iss' claims)"
+  default     = "https://kubernetes.default.svc.cluster.local"
+}

versions.tf

@@ -1,10 +1,9 @@
 # Terraform version and plugin versions
 
 terraform {
-  required_version = "~> 0.12.0"
+  required_version = ">= 0.13.0, < 2.0.0"
   required_providers {
-    local    = "~> 1.2"
-    template = "~> 2.1"
-    tls      = "~> 2.0"
+    random = "~> 3.1"
+    tls    = "~> 4.0"
   }
 }