Update kube-apiserver runtime-config flags

* MutatingAdmissionPolicy is available as a beta and alpha API

.github/dependabot.yaml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily"

.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  terraform:
+    name: fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v6
+
+      - name: terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.11.1
+
+      - name: fmt
+        run: terraform fmt -check -diff -recursive

README.md

@@ -1,27 +1,29 @@
-# terraform-render-bootkube
+# terraform-render-bootstrap
+[![Workflow](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml/badge.svg)](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml?query=branch%3Amain)
+[![Sponsors](https://img.shields.io/github/sponsors/poseidon?logo=github)](https://github.com/sponsors/poseidon)
+[![Mastodon](https://img.shields.io/badge/follow-news-6364ff?logo=mastodon)](https://fosstodon.org/@typhoon)
 
-`terraform-render-bootkube` is a Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) assets for bootstrapping a Kubernetes cluster.
+`terraform-render-bootstrap` is a Terraform module that renders TLS certificates, static pods, and manifests for bootstrapping a Kubernetes cluster.
 
 ## Audience
 
-`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+`terraform-render-bootstrap` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootstrap module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
-Use the module to declare bootkube assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
+Use the module to declare bootstrap assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
 
 ```hcl
-module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=SHA"
+module "bootstrap" {
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=SHA"
 
   cluster_name = "example"
   api_servers = ["node1.example.com"]
   etcd_servers = ["node1.example.com"]
-  asset_dir = "/home/core/clusters/mycluster"
 }
 ```
 
-Generate the assets.
+Generate assets in Terraform state.
 
 ```sh
 terraform init
@@ -29,21 +31,12 @@ terraform plan
 terraform apply
 ```
 
-Find bootkube assets rendered to the `asset_dir` path. That's it.
+To inspect and write assets locally (e.g. debugging) use the `assets_dist` Terraform output.
 
-### Comparison
-
-Render bootkube assets directly with bootkube v0.14.0.
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:6443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
 ```
-
-Compare assets. Rendered assets may differ slightly from bootkube assets to reflect decisions made by the [Typhoon](https://github.com/poseidon/typhoon) distribution.
-
-```sh
-pushd /home/core/mycluster
-mv manifests-networking/* manifests
-popd
-diff -rw assets /home/core/mycluster
+resource local_file "assets" {
+  for_each = module.bootstrap.assets_dist
+  filename = "some-assets/${each.key}"
+  content = each.value
+}
 ```

assets.tf

@@ -1,110 +0,0 @@
-# Self-hosted Kubernetes bootstrap-manifests
-resource "template_dir" "bootstrap-manifests" {
-  source_dir      = "${path.module}/resources/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/bootstrap-manifests"
-
-  vars {
-    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-
-    cloud_provider = "${var.cloud_provider}"
-    pod_cidr       = "${var.pod_cidr}"
-    service_cidr   = "${var.service_cidr}"
-
-    trusted_certs_dir = "${var.trusted_certs_dir}"
-    apiserver_port    = "${var.apiserver_port}"
-  }
-}
-
-# Self-hosted Kubernetes manifests
-resource "template_dir" "manifests" {
-  source_dir      = "${path.module}/resources/manifests"
-  destination_dir = "${var.asset_dir}/manifests"
-
-  vars {
-    hyperkube_image        = "${var.container_images["hyperkube"]}"
-    pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}"
-    coredns_image          = "${var.container_images["coredns"]}"
-
-    etcd_servers           = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-    control_plane_replicas = "${max(2, length(var.etcd_servers))}"
-
-    cloud_provider         = "${var.cloud_provider}"
-    pod_cidr               = "${var.pod_cidr}"
-    service_cidr           = "${var.service_cidr}"
-    cluster_domain_suffix  = "${var.cluster_domain_suffix}"
-    cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
-    trusted_certs_dir      = "${var.trusted_certs_dir}"
-    apiserver_port         = "${var.apiserver_port}"
-
-    ca_cert            = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    ca_key             = "${base64encode(tls_private_key.kube-ca.private_key_pem)}"
-    server             = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-    apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
-    apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
-    serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
-    serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
-
-    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
-
-    aggregation_flags       = "${var.enable_aggregation == "true" ? indent(8, local.aggregation_flags) : ""}"
-    aggregation_ca_cert     = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""}"
-    aggregation_client_cert = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""}"
-    aggregation_client_key  = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""}"
-  }
-}
-
-locals {
-  aggregation_flags = <<EOF
-
-- --proxy-client-cert-file=/etc/kubernetes/secrets/aggregation-client.crt
-- --proxy-client-key-file=/etc/kubernetes/secrets/aggregation-client.key
-- --requestheader-client-ca-file=/etc/kubernetes/secrets/aggregation-ca.crt
-- --requestheader-extra-headers-prefix=X-Remote-Extra-
-- --requestheader-group-headers=X-Remote-Group
-- --requestheader-username-headers=X-Remote-UserEOF
-}
-
-# Generated kubeconfig for Kubelets
-resource "local_file" "kubeconfig-kubelet" {
-  content  = "${data.template_file.kubeconfig-kubelet.rendered}"
-  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
-}
-
-# Generated admin kubeconfig (bootkube requires it be at auth/kubeconfig)
-# https://github.com/kubernetes-incubator/bootkube/blob/master/pkg/bootkube/bootkube.go#L42
-resource "local_file" "kubeconfig-admin" {
-  content  = "${data.template_file.kubeconfig-admin.rendered}"
-  filename = "${var.asset_dir}/auth/kubeconfig"
-}
-
-# Generated admin kubeconfig in a file named after the cluster
-resource "local_file" "kubeconfig-admin-named" {
-  content  = "${data.template_file.kubeconfig-admin.rendered}"
-  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
-}
-
-data "template_file" "kubeconfig-kubelet" {
-  template = "${file("${path.module}/resources/kubeconfig-kubelet")}"
-
-  vars {
-    ca_cert      = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-    server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-  }
-}
-
-data "template_file" "kubeconfig-admin" {
-  template = "${file("${path.module}/resources/kubeconfig-admin")}"
-
-  vars {
-    name         = "${var.cluster_name}"
-    ca_cert      = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.admin.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.admin.private_key_pem)}"
-    server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-  }
-}

auth.tf

@@ -0,0 +1,68 @@
+locals {
+  # component kubeconfigs assets map
+  auth_kubeconfigs = {
+    "auth/admin.conf"              = local.kubeconfig-admin,
+    "auth/controller-manager.conf" = local.kubeconfig-controller-manager
+    "auth/scheduler.conf"          = local.kubeconfig-scheduler
+  }
+}
+
+locals {
+  # Generated admin kubeconfig to bootstrap control plane
+  kubeconfig-admin = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-controller-manager = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.controller-manager.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.controller-manager.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-scheduler = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.scheduler.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.scheduler.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kubeconfig to bootstrap Kubelets
+  kubeconfig-bootstrap = templatefile("${path.module}/resources/kubeconfig-bootstrap",
+    {
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+      token_id     = random_password.bootstrap-token-id.result
+      token_secret = random_password.bootstrap-token-secret.result
+    }
+  )
+}
+
+# Generate a cryptographically random token id (public)
+resource "random_password" "bootstrap-token-id" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret
+resource "random_password" "bootstrap-token-secret" {
+  length  = 16
+  upper   = false
+  special = false
+}
+

conditional.tf

@@ -1,47 +1,36 @@
 # Assets generated only when certain options are chosen
 
-resource "template_dir" "flannel-manifests" {
-  count           = "${var.networking == "flannel" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/flannel"
-  destination_dir = "${var.asset_dir}/manifests-networking"
+locals {
+  # flannel manifests map
+  # { manifests-networking/manifest.yaml => content }
+  flannel_manifests = {
+    for name in fileset("${path.module}/resources/flannel", "*.yaml") :
+    "manifests/network/${name}" => templatefile(
+      "${path.module}/resources/flannel/${name}",
+      {
+        flannel_image         = var.container_images["flannel"]
+        flannel_cni_image     = var.container_images["flannel_cni"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
+      }
+    )
+    if var.components.enable && var.components.flannel.enable && var.networking == "flannel"
+  }
 
-  vars {
-    flannel_image     = "${var.container_images["flannel"]}"
-    flannel_cni_image = "${var.container_images["flannel_cni"]}"
-
-    pod_cidr = "${var.pod_cidr}"
+  # cilium manifests map
+  # { manifests-networking/manifest.yaml => content }
+  cilium_manifests = {
+    for name in fileset("${path.module}/resources/cilium", "**/*.yaml") :
+    "manifests/network/${name}" => templatefile(
+      "${path.module}/resources/cilium/${name}",
+      {
+        cilium_agent_image    = var.container_images["cilium_agent"]
+        cilium_operator_image = var.container_images["cilium_operator"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
+      }
+    )
+    if var.components.enable && var.components.cilium.enable && var.networking == "cilium"
   }
 }
 
-resource "template_dir" "calico-manifests" {
-  count           = "${var.networking == "calico" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/calico"
-  destination_dir = "${var.asset_dir}/manifests-networking"
-
-  vars {
-    calico_image     = "${var.container_images["calico"]}"
-    calico_cni_image = "${var.container_images["calico_cni"]}"
-
-    network_mtu                     = "${var.network_mtu}"
-    network_encapsulation           = "${indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")}"
-    ipip_enabled                   = "${var.network_encapsulation == "ipip" ? true : false}"
-    ipip_readiness                 = "${var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""}"
-    vxlan_enabled                   = "${var.network_encapsulation == "vxlan" ? true : false}"
-    network_ip_autodetection_method = "${var.network_ip_autodetection_method}"
-    pod_cidr                        = "${var.pod_cidr}"
-    enable_reporting                = "${var.enable_reporting}"
-  }
-}
-
-resource "template_dir" "kube-router-manifests" {
-  count           = "${var.networking == "kube-router" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/kube-router"
-  destination_dir = "${var.asset_dir}/manifests-networking"
-
-  vars {
-    kube_router_image = "${var.container_images["kube_router"]}"
-    flannel_cni_image = "${var.container_images["flannel_cni"]}"
-
-    network_mtu = "${var.network_mtu}"
-  }
-}

manifests.tf

@@ -0,0 +1,77 @@
+locals {
+  # Kubernetes static pod manifests map
+  # {static-manifests/manifest.yaml => content }
+  static_manifests = {
+    for name in fileset("${path.module}/resources/static-manifests", "*.yaml") :
+    "static-manifests/${name}" => templatefile(
+      "${path.module}/resources/static-manifests/${name}",
+      {
+        kube_apiserver_image          = var.container_images["kube_apiserver"]
+        kube_controller_manager_image = var.container_images["kube_controller_manager"]
+        kube_scheduler_image          = var.container_images["kube_scheduler"]
+
+        etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers))
+        pod_cidr     = var.pod_cidr
+        service_cidr = var.service_cidr
+
+        service_account_issuer = var.service_account_issuer
+        aggregation_flags      = var.enable_aggregation ? indent(4, local.aggregation_flags) : ""
+      }
+    )
+  }
+
+  # Kubernetes control plane manifests map
+  # { manifests/manifest.yaml => content }
+  manifests = merge({
+    for name in fileset("${path.module}/resources/manifests", "**/*.yaml") :
+    "manifests/${name}" => templatefile(
+      "${path.module}/resources/manifests/${name}",
+      {
+        server         = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        apiserver_host = var.api_servers[0]
+        apiserver_port = var.external_apiserver_port
+        token_id       = random_password.bootstrap-token-id.result
+        token_secret   = random_password.bootstrap-token-secret.result
+      }
+    )
+    },
+    # CoreDNS manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/coredns", "*.yaml") :
+      "manifests/coredns/${name}" => templatefile(
+        "${path.module}/resources/coredns/${name}",
+        {
+          coredns_image          = var.container_images["coredns"]
+          control_plane_replicas = max(2, length(var.etcd_servers))
+          cluster_domain_suffix  = var.cluster_domain_suffix
+          cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+        }
+      ) if var.components.enable && var.components.coredns.enable
+    },
+    # kube-proxy manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/kube-proxy", "*.yaml") :
+      "manifests/kube-proxy/${name}" => templatefile(
+        "${path.module}/resources/kube-proxy/${name}",
+        {
+          kube_proxy_image      = var.container_images["kube_proxy"]
+          pod_cidr              = var.pod_cidr
+          daemonset_tolerations = var.daemonset_tolerations
+        }
+      ) if var.components.enable && var.components.kube_proxy.enable && var.networking != "cilium"
+    }
+  )
+}
+
+locals {
+  aggregation_flags = <<EOF
+
+- --proxy-client-cert-file=/etc/kubernetes/pki/aggregation-client.crt
+- --proxy-client-key-file=/etc/kubernetes/pki/aggregation-client.key
+- --requestheader-client-ca-file=/etc/kubernetes/pki/aggregation-ca.crt
+- --requestheader-extra-headers-prefix=X-Remote-Extra-
+- --requestheader-group-headers=X-Remote-Group
+- --requestheader-username-headers=X-Remote-User
+EOF
+}
+

outputs.tf

@@ -1,71 +1,76 @@
-output "id" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
-}
-
-output "content_hash" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
-}
 
 output "cluster_dns_service_ip" {
-  value = "${cidrhost(var.service_cidr, 10)}"
+  value = cidrhost(var.service_cidr, 10)
 }
 
 // Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
 output "kubeconfig-kubelet" {
-  value = "${data.template_file.kubeconfig-kubelet.rendered}"
+  value     = local.kubeconfig-bootstrap
+  sensitive = true
 }
 
 // Generated kubeconfig for admins (i.e. human super-user)
 output "kubeconfig-admin" {
-  value = "${data.template_file.kubeconfig-admin.rendered}"
+  value     = local.kubeconfig-admin
+  sensitive = true
+}
+
+# assets to distribute to controllers
+# { some/path => content }
+output "assets_dist" {
+  # combine maps of assets
+  value = merge(
+    local.auth_kubeconfigs,
+    local.etcd_tls,
+    local.kubernetes_tls,
+    local.aggregation_tls,
+    local.static_manifests,
+    local.manifests,
+    local.flannel_manifests,
+    local.cilium_manifests,
+  )
+  sensitive = true
 }
 
 # etcd TLS assets
 
 output "etcd_ca_cert" {
-  value = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  value     = tls_self_signed_cert.etcd-ca.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_cert" {
-  value = "${tls_locally_signed_cert.client.cert_pem}"
+  value     = tls_locally_signed_cert.client.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_key" {
-  value = "${tls_private_key.client.private_key_pem}"
+  value     = tls_private_key.client.private_key_pem
+  sensitive = true
 }
 
 output "etcd_server_cert" {
-  value = "${tls_locally_signed_cert.server.cert_pem}"
+  value     = tls_locally_signed_cert.server.cert_pem
+  sensitive = true
 }
 
 output "etcd_server_key" {
-  value = "${tls_private_key.server.private_key_pem}"
+  value     = tls_private_key.server.private_key_pem
+  sensitive = true
 }
 
 output "etcd_peer_cert" {
-  value = "${tls_locally_signed_cert.peer.cert_pem}"
+  value     = tls_locally_signed_cert.peer.cert_pem
+  sensitive = true
 }
 
 output "etcd_peer_key" {
-  value = "${tls_private_key.peer.private_key_pem}"
+  value     = tls_private_key.peer.private_key_pem
+  sensitive = true
 }
 
-# Some platforms may need to reconstruct the kubeconfig directly in user-data.
-# That can't be done with the way template_file interpolates multi-line
-# contents so the raw components of the kubeconfig may be needed.
+# Kubernetes TLS assets
 
-output "ca_cert" {
-  value = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-}
-
-output "kubelet_cert" {
-  value = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-}
-
-output "kubelet_key" {
-  value = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-}
-
-output "server" {
-  value = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
+output "service_account_public_key" {
+  value = tls_private_key.service-account.public_key_pem
 }

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -1,56 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-apiserver
-  namespace: kube-system
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-spec:
-  hostNetwork: true
-  containers:
-  - name: kube-apiserver
-    image: ${hyperkube_image}
-    command:
-    - /hyperkube
-    - apiserver
-    - --advertise-address=$(POD_IP)
-    - --allow-privileged=true
-    - --anonymous-auth=false
-    - --authorization-mode=RBAC
-    - --bind-address=0.0.0.0
-    - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --cloud-provider=${cloud_provider}
-    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-    - --etcd-servers=${etcd_servers}
-    - --insecure-port=0
-    - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-    - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-    - --secure-port=${apiserver_port}
-    - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
-    - --service-cluster-ip-range=${service_cidr}
-    - --storage-backend=etcd3
-    - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-    - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
-    env:
-    - name: POD_IP
-      valueFrom:
-        fieldRef:
-          fieldPath: status.podIP
-    volumeMounts:
-    - name: secrets
-      mountPath: /etc/kubernetes/secrets
-      readOnly: true
-    - name: ssl-certs-host
-      mountPath: /etc/ssl/certs
-      readOnly: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
-  - name: ssl-certs-host
-    hostPath:
-      path: ${trusted_certs_dir}

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -1,40 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-controller-manager
-  namespace: kube-system
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-spec:
-  containers:
-  - name: kube-controller-manager
-    image: ${hyperkube_image}
-    command:
-    - ./hyperkube
-    - controller-manager
-    - --allocate-node-cidrs=true
-    - --cluster-cidr=${pod_cidr}
-    - --service-cluster-ip-range=${service_cidr}
-    - --cloud-provider=${cloud_provider}
-    - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
-    - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
-    - --configure-cloud-routes=false
-    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
-    - --leader-elect=true
-    - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
-    volumeMounts:
-    - name: secrets
-      mountPath: /etc/kubernetes/secrets
-      readOnly: true
-    - name: ssl-host
-      mountPath: /etc/ssl/certs
-      readOnly: true
-  hostNetwork: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
-  - name: ssl-host
-    hostPath:
-      path: ${trusted_certs_dir}

resources/bootstrap-manifests/bootstrap-scheduler.yaml

@@ -1,25 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-scheduler
-  namespace: kube-system
-  annotations:
-    seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-spec:
-  containers:
-  - name: kube-scheduler
-    image: ${hyperkube_image}
-    command:
-    - ./hyperkube
-    - scheduler
-    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
-    - --leader-elect=true
-    volumeMounts:
-    - name: secrets
-      mountPath: /etc/kubernetes/secrets
-      readOnly: true
-  hostNetwork: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets

resources/calico/bgpconfigurations-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: bgpconfigurations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BGPConfiguration
-    plural: bgpconfigurations
-    singular: bgpconfiguration

resources/calico/bgppeers-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: bgppeers.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BGPPeer
-    plural: bgppeers
-    singular: bgppeer

resources/calico/blockaffinities-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: blockaffinities.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BlockAffinity
-    plural: blockaffinities
-    singular: blockaffinity

resources/calico/cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: calico-node
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: calico-node
-subjects:
-- kind: ServiceAccount
-  name: calico-node
-  namespace: kube-system

resources/calico/cluster-role.yaml

@@ -1,108 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-node
-rules:
-  - apiGroups: [""]
-    resources:
-      - pods
-      - nodes
-      - namespaces
-    verbs:
-      - get
-  - apiGroups: [""]
-    resources:
-      - endpoints
-      - services
-    verbs:
-      - watch
-      - list
-  # Used by Calico for policy information
-  - apiGroups: [""]
-    resources:
-      - pods
-      - namespaces
-      - serviceaccounts
-    verbs:
-      - list
-      - watch
-  - apiGroups: [""]
-    resources:
-      - nodes/status
-    verbs:
-      # Calico patches the node NetworkUnavilable status
-      - patch
-      # Calico updates some info in node annotations
-      - update
-  # CNI plugin patches pods/status
-  - apiGroups: [""]
-    resources:
-      - pods/status
-    verbs:
-      - patch
-  # Calico reads some info on nodes
-  - apiGroups: [""]
-    resources:
-      - nodes
-    verbs:
-      - get
-      - list
-      - watch
-  # Calico monitors Kubernetes NetworkPolicies
-  - apiGroups: ["networking.k8s.io"]
-    resources:
-      - networkpolicies
-    verbs:
-      - watch
-      - list
-  # Calico monitors its CRDs
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - globalfelixconfigs
-      - felixconfigurations
-      - bgppeers
-      - globalbgpconfigs
-      - bgpconfigurations
-      - ippools
-      - ipamblocks
-      - globalnetworkpolicies
-      - globalnetworksets
-      - networksets
-      - networkpolicies
-      - clusterinformations
-      - hostendpoints
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - felixconfigurations
-      - ippools
-      - clusterinformations
-    verbs:
-      - create
-      - update
-  # Calico may perform IPAM allocations
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - blockaffinities
-      - ipamblocks
-      - ipamhandles
-    verbs:
-      - get
-      - list
-      - create
-      - update
-      - delete
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - ipamconfigs
-    verbs:
-      - get
-  # Watch block affinities for route aggregation
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - blockaffinities
-    verbs:
-      - watch

resources/calico/clusterinformations-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: clusterinformations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: ClusterInformation
-    plural: clusterinformations
-    singular: clusterinformation

resources/calico/config.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: calico-config
-  namespace: kube-system
-data:
-  # Disable Typha for now.
-  typha_service_name: "none"
-  # Calico backend to use
-  calico_backend: "bird"
-  # Calico MTU
-  veth_mtu: "${network_mtu}"
-  # The CNI network configuration to install on each node.
-  cni_network_config: |-
-    {
-      "name": "k8s-pod-network",
-      "cniVersion": "0.3.1",
-      "plugins": [
-        {
-          "type": "calico",
-          "log_level": "info",
-          "datastore_type": "kubernetes",
-          "nodename": "__KUBERNETES_NODE_NAME__",
-          "mtu": __CNI_MTU__,
-          "ipam": {
-            "type": "calico-ipam"
-          },
-          "policy": {
-            "type": "k8s"
-          },
-          "kubernetes": {
-            "kubeconfig": "__KUBECONFIG_FILEPATH__"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {"portMappings": true}
-        }
-      ]
-    }

resources/calico/daemonset.yaml

@@ -1,191 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    k8s-app: calico-node
-spec:
-  selector:
-    matchLabels:
-      k8s-app: calico-node
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: calico-node
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      priorityClassName: system-node-critical
-      serviceAccountName: calico-node
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      initContainers:
-        # Install Calico CNI binaries and CNI network config file on nodes
-        - name: install-cni
-          image: ${calico_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            # Name of the CNI config file to create on each node.
-            - name: CNI_CONF_NAME
-              value: "10-calico.conflist"
-            # Set node name based on k8s nodeName
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Contents of the CNI config to create on each node.
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-            - name: CNI_NET_DIR
-              value: "/etc/kubernetes/cni/net.d"
-            - name: CNI_MTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            - name: SLEEP
-              value: "false"
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
-      containers:
-        - name: calico-node
-          image: ${calico_image}
-          env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Wait for datastore
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # Typha support: controlled by the ConfigMap.
-            - name: FELIX_TYPHAK8SSERVICENAME
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: typha_service_name
-            - name: FELIX_USAGEREPORTINGENABLED
-              value: "${enable_reporting}"
-            # Set node name based on k8s nodeName.
-            - name: NODENAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            # Calico network backend
-            - name: CALICO_NETWORKING_BACKEND
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: calico_backend
-            # Cluster type to identify the deployment type
-            - name: CLUSTER_TYPE
-              value: "k8s,bgp"
-            # Auto-detect the BGP IP address.
-            - name: IP
-              value: "autodetect"
-            - name: IP_AUTODETECTION_METHOD
-              value: "${network_ip_autodetection_method}"
-            # Whether Felix should enable IP-in-IP tunnel
-            - name: FELIX_IPINIPENABLED
-              value: "${ipip_enabled}"
-            # MTU to set on the IPIP tunnel (if enabled)
-            - name: FELIX_IPINIPMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            # Whether Felix should enable VXLAN tunnel
-            - name: FELIX_VXLANENABLED
-              value: "${vxlan_enabled}"
-            # MTU to set on the VXLAN tunnel (if enabled)
-            - name: FELIX_VXLANMTU
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: veth_mtu
-            - name: NO_DEFAULT_POOLS
-              value: "true"
-            # Disable file logging so `kubectl logs` works.
-            - name: CALICO_DISABLE_FILE_LOGGING
-              value: "true"
-            # Set Felix endpoint to host default action to ACCEPT.
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
-            # Disable IPV6 on Kubernetes.
-            - name: FELIX_IPV6SUPPORT
-              value: "false"
-            # Enable felix info logging.
-            - name: FELIX_LOGSEVERITYSCREEN
-              value: "info"
-            - name: FELIX_HEALTHENABLED
-              value: "true"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 150m
-          livenessProbe:
-            httpGet:
-              path: /liveness
-              port: 9099
-              host: localhost
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-          readinessProbe:
-            exec:
-              command:
-                - /bin/calico-node
-                - -felix-ready
-                ${ipip_readiness}
-            periodSeconds: 10
-          volumeMounts:
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: var-lib-calico
-              mountPath: /var/lib/calico
-              readOnly: false
-            - name: var-run-calico
-              mountPath: /var/run/calico
-              readOnly: false
-            - name: xtables-lock
-              mountPath: /run/xtables.lock
-              readOnly: false
-      terminationGracePeriodSeconds: 0
-      volumes:
-        # Used by calico/node
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-lib-calico
-          hostPath:
-            path: /var/lib/calico
-        - name: var-run-calico
-          hostPath:
-            path: /var/run/calico
-        - name: xtables-lock
-          hostPath:
-            type: FileOrCreate
-            path: /run/xtables.lock
-        # Used by install-cni
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-conf-dir
-          hostPath:
-            path: /etc/kubernetes/cni/net.d

resources/calico/default-ipv4-ippool.yaml

@@ -1,10 +0,0 @@
-apiVersion: crd.projectcalico.org/v1
-kind: IPPool
-metadata:
-  name: default-ipv4-ippool
-spec:
-  blockSize: 24
-  cidr: ${pod_cidr}
-  ${network_encapsulation}
-  natOutgoing: true
-  nodeSelector: all()

resources/calico/felixconfigurations-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: felixconfigurations.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: FelixConfiguration
-    plural: felixconfigurations
-    singular: felixconfiguration

resources/calico/globalnetworkpolicies-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: globalnetworkpolicies.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalNetworkPolicy
-    plural: globalnetworkpolicies
-    singular: globalnetworkpolicy

resources/calico/globalnetworksets-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: globalnetworksets.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalNetworkSet
-    plural: globalnetworksets
-    singular: globalnetworkset

resources/calico/hostendpoints-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: hostendpoints.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: HostEndpoint
-    plural: hostendpoints
-    singular: hostendpoint

resources/calico/ipamblocks.crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamblocks.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMBlock
-    plural: ipamblocks
-    singular: ipamblock

resources/calico/ipamconfigs-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMConfig
-    plural: ipamconfigs
-    singular: ipamconfig

resources/calico/ipamhandles-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ipamhandles.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPAMHandle
-    plural: ipamhandles
-    singular: ipamhandle

resources/calico/ippools-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: ippools.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPPool
-    plural: ippools
-    singular: ippool

resources/calico/networkpolicies-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: networkpolicies.crd.projectcalico.org
-spec:
-  scope: Namespaced
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: NetworkPolicy
-    plural: networkpolicies
-    singular: networkpolicy

resources/calico/networksets-crd.yaml

@@ -1,12 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-kind: CustomResourceDefinition
-metadata:
-  name: networksets.crd.projectcalico.org
-spec:
-  scope: Namespaced
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: NetworkSet
-    plural: networksets
-    singular: networkset

resources/cilium/cluster-role-binding.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-agent
+subjects:
+- kind: ServiceAccount
+  name: cilium-agent
+  namespace: kube-system
+

resources/cilium/cluster-role.yaml

@@ -0,0 +1,188 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  # to automatically delete [core|kube]dns pods so that are starting to being
+  # managed by Cilium
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  # to perform LB IP allocation for BGP
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+  - services
+  - endpoints
+  # to check apiserver connectivity
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumendpoints/finalizers
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumnodes/finalizers
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
+  - ciliumendpointslices
+  - ciliumloadbalancerippools
+  - ciliumloadbalancerippools/status
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
+# Cilium leader elects if among multiple operator replicas
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-agent
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - pods
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumegressnatpolicies
+  - ciliumendpointslices
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+

resources/cilium/config.yaml

@@ -0,0 +1,175 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium
+  namespace: kube-system
+data:
+  # Identity allocation mode selects how identities are shared between cilium
+  # nodes by setting how they are stored. The options are "crd" or "kvstore".
+  # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+  #   These can be queried with:
+  #     kubectl get ciliumid
+  # - "kvstore" stores identities in a kvstore, etcd or consul, that is
+  #   configured below. Cilium versions before 1.6 supported only the kvstore
+  #   backend. Upgrades from these older cilium versions should continue using
+  #   the kvstore by commenting out the identity-allocation-mode below, or
+  #   setting it to "kvstore".
+  identity-allocation-mode: crd
+  cilium-endpoint-gc-interval: "5m0s"
+  nodes-gc-interval: "5m0s"
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "false"
+  # The agent can be put into the following three policy enforcement modes
+  # default, always and never.
+  # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes
+  enable-policy: "default"
+
+  # Prometheus
+  # enable-metrics: "true"
+  # prometheus-serve-addr: ":foo"
+  # operator-prometheus-serve-addr: ":bar"
+
+  # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
+  # address.
+  enable-ipv4: "true"
+
+  # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
+  # address.
+  enable-ipv6: "false"
+
+  # Enable probing for a more efficient clock source for the BPF datapath
+  enable-bpf-clock-probe: "true"
+
+  # Enable use of transparent proxying mechanisms (Linux 5.7+)
+  enable-bpf-tproxy: "false"
+
+  # If you want cilium monitor to aggregate tracing for packets, set this level
+  # to "low", "medium", or "maximum". The higher the level, the less packets
+  # that will be seen in monitor output.
+  monitor-aggregation: medium
+
+  # The monitor aggregation interval governs the typical time between monitor
+  # notification events for each allowed connection.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-interval: 5s
+
+  # The monitor aggregation flags determine which TCP flags which, upon the
+  # first observation, cause monitor notifications to be generated.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-flags: all
+
+  # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+  # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+  bpf-map-dynamic-size-ratio: "0.0025"
+  # bpf-policy-map-max specified the maximum number of entries in endpoint
+  # policy map (per endpoint)
+  bpf-policy-map-max: "16384"
+  # bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
+  # backend and affinity maps.
+  bpf-lb-map-max: "65536"
+
+  # Pre-allocation of map entries allows per-packet latency to be reduced, at
+  # the expense of up-front memory allocation for the entries in the maps. The
+  # default value below will minimize memory usage in the default installation;
+  # users who are sensitive to latency may consider setting this to "true".
+  #
+  # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+  # this option and behave as though it is set to "true".
+  #
+  # If this value is modified, then during the next Cilium startup the restore
+  # of existing endpoints and tracking of ongoing connections may be disrupted.
+  # As a result, reply packets may be dropped and the load-balancing decisions
+  # for established connections may change.
+  #
+  # If this option is set to "false" during an upgrade from 1.3 or earlier to
+  # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+  preallocate-bpf-maps: "false"
+
+  # Name of the cluster. Only relevant when building a mesh of clusters.
+  cluster-name: default
+  # Unique ID of the cluster. Must be unique across all conneted clusters and
+  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+  cluster-id: "0"
+
+  # Encapsulation mode for communication between nodes
+  # Possible values:
+  #   - disabled
+  #   - vxlan (default)
+  #   - geneve
+  routing-mode: "tunnel"
+  tunnel: vxlan
+  # Enables L7 proxy for L7 policy enforcement and visibility
+  enable-l7-proxy: "true"
+
+  auto-direct-node-routes: "false"
+
+  # enableXTSocketFallback enables the fallback compatibility solution
+  # when the xt_socket kernel module is missing and it is needed for
+  # the datapath L7 redirection to work properly.  See documentation
+  # for details on when this can be disabled:
+  # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version.
+  enable-xt-socket-fallback: "true"
+
+  # installIptablesRules enables installation of iptables rules to allow for
+  # TPROXY (L7 proxy injection), itpables based masquerading and compatibility
+  # with kube-proxy. See documentation for details on when this can be
+  # disabled.
+  install-iptables-rules: "true"
+
+  # masquerade traffic leaving the node destined for outside
+  enable-ipv4-masquerade: "true"
+  enable-ipv6-masquerade: "false"
+
+  # bpfMasquerade enables masquerading with BPF instead of iptables
+  enable-bpf-masquerade: "true"
+
+  # kube-proxy
+  kube-proxy-replacement:  "true"
+  kube-proxy-replacement-healthz-bind-address: ":10256"
+  enable-session-affinity: "true"
+
+  # ClusterIPs from host namespace
+  bpf-lb-sock: "true"
+  # ClusterIPs from external nodes
+  bpf-lb-external-clusterip: "true"
+
+  # NodePort
+  enable-node-port: "true"
+  enable-health-check-nodeport: "false"
+
+  # ExternalIPs
+  enable-external-ips: "true"
+
+  # HostPort
+  enable-host-port: "true"
+
+  # IPAM
+  ipam: "cluster-pool"
+  disable-cnp-status-updates: "true"
+  cluster-pool-ipv4-cidr: "${pod_cidr}"
+  cluster-pool-ipv4-mask-size: "24"
+
+  # Health
+  agent-health-port: "9876"
+  enable-health-checking: "true"
+  enable-endpoint-health-checking: "true"
+
+  # Identity
+  enable-well-known-identities: "false"
+  enable-remote-node-identity: "true"
+
+  # Misc
+  enable-bandwidth-manager: "false"
+  enable-local-redirect-policy: "false"
+  policy-audit-mode: "false"
+  operator-api-serve-addr: "127.0.0.1:9234"
+  enable-l2-neigh-discovery: "true"
+  enable-k8s-terminating-endpoint: "true"
+  enable-k8s-networkpolicy: "true"
+  external-envoy-proxy: "false"
+  write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+  cni-exclusive: "true"
+  cni-log-file: "/var/run/cilium/cilium-cni.log"

resources/cilium/daemonset.yaml

@@ -0,0 +1,219 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cilium
+  namespace: kube-system
+  labels:
+    k8s-app: cilium
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium-agent
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: cilium-agent
+    spec:
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      serviceAccountName: cilium-agent
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+      # Cilium v1.13.1 starts installing CNI plugins in yet another init container
+      # https://github.com/cilium/cilium/pull/24075
+      - name: install-cni
+        image: ${cilium_agent_image}
+        command:
+          - /install-plugin.sh
+        securityContext:
+          privileged: true
+          capabilities:
+            drop:
+              - ALL
+        volumeMounts:
+          - name: cni-bin-dir
+            mountPath: /host/opt/cni/bin
+
+      # Required to mount cgroup2 filesystem on the underlying Kubernetes node.
+      # We use nsenter command with host's cgroup and mount namespaces enabled.
+      - name: mount-cgroup
+        image: ${cilium_agent_image}
+        command:
+          - sh
+          - -ec
+          # The statically linked Go program binary is invoked to avoid any
+          # dependency on utilities like sh and mount that can be missing on certain
+          # distros installed on the underlying host. Copy the binary to the
+          # same directory where we install cilium cni plugin so that exec permissions
+          # are available.
+          - 'cp /usr/bin/cilium-mount /hostbin/cilium-mount && nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "$${BIN_PATH}/cilium-mount" $CGROUP_ROOT; rm /hostbin/cilium-mount'
+        env:
+          - name: CGROUP_ROOT
+            value: /run/cilium/cgroupv2
+          - name: BIN_PATH
+            value: /opt/cni/bin
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: hostproc
+            mountPath: /hostproc
+          - name: cni-bin-dir
+            mountPath: /hostbin
+
+      - name: clean-cilium-state
+        image: ${cilium_agent_image}
+        command:
+        - /init-container.sh
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        # Required to mount cgroup filesystem from the host to cilium agent pod
+        - name: cilium-cgroup
+          mountPath: /run/cilium/cgroupv2
+          mountPropagation: HostToContainer
+
+      containers:
+      - name: cilium-agent
+        image: ${cilium_agent_image}
+        command:
+        - cilium-agent
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        ports:
+          # Not yet used, prefer exec's
+          - name: health
+            protocol: TCP
+            containerPort: 9876
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        securityContext:
+          privileged: true
+        livenessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 30
+          initialDelaySeconds: 120
+          successThreshold: 1
+          failureThreshold: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 20
+          initialDelaySeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+          timeoutSeconds: 5
+        volumeMounts:
+        # Load kernel modules
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+        # Keep state between restarts
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
+        # Configuration
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+        # Install config on host
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      terminationGracePeriodSeconds: 1
+      volumes:
+      # Load kernel modules
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      # Access iptables concurrently with other processes (e.g. kube-proxy)
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock
+      # Keep state between restarts
+      - name: var-run-cilium
+        hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+      # Keep state between restarts for bpf maps
+      - name: sys-fs-bpf
+        hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+      # Mount host cgroup2 filesystem
+      - name: hostproc
+        hostPath:
+          path: /proc
+          type: Directory
+      - name: cilium-cgroup
+        hostPath:
+          path: /run/cilium/cgroupv2
+          type: DirectoryOrCreate
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium
+      # Install CNI plugin and config on host
+      - name: cni-bin-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path:  /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path: /etc/cni/net.d
+

resources/cilium/deployment.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      name: cilium-operator
+  template:
+    metadata:
+      labels:
+        name: cilium-operator
+    spec:
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      serviceAccountName: cilium-operator
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      containers:
+      - name: cilium-operator
+        image: ${cilium_operator_image}
+        command:
+        - cilium-operator-generic
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        - --debug=$(CILIUM_DEBUG)
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        - name: CILIUM_DEBUG
+          valueFrom:
+            configMapKeyRef:
+              name: cilium
+              key: debug
+              optional: true
+        ports:
+          - name: health
+            protocol: TCP
+            containerPort: 9234
+        livenessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 3
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          periodSeconds: 15
+          timeoutSeconds: 3
+          failureThreshold: 5
+        volumeMounts:
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+      topologySpreadConstraints:
+        - topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              name: cilium-operator
+          maxSkew: 1
+          whenUnsatisfiable: DoNotSchedule
+      volumes:
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium

resources/cilium/service-account.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-agent
+  namespace: kube-system
+

resources/coredns/cluster-role.yaml

@@ -14,6 +14,12 @@ rules:
     verbs:
       - list
       - watch
+  - apiGroups: ["discovery.k8s.io"]
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
   - apiGroups: [""]
     resources:
       - nodes

resources/coredns/config.yaml

@@ -7,7 +7,9 @@ data:
   Corefile: |
     .:53 {
         errors
-        health
+        health {
+          lameduck 5s
+        }
         ready
         log . {
             class error

resources/coredns/deployment.yaml

@@ -6,7 +6,6 @@ metadata:
   labels:
     k8s-app: coredns
     kubernetes.io/name: "CoreDNS"
-    kubernetes.io/cluster-service: "true"
 spec:
   replicas: ${control_plane_replicas}
   strategy:
@@ -22,10 +21,15 @@ spec:
       labels:
         tier: control-plane
         k8s-app: coredns
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node.kubernetes.io/controller
+                operator: Exists
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
@@ -42,9 +46,12 @@ spec:
                   - coredns
               topologyKey: kubernetes.io/hostname
       priorityClassName: system-cluster-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: coredns
       tolerations:
-        - key: node-role.kubernetes.io/master
+        - key: node-role.kubernetes.io/controller
           effect: NoSchedule
       containers:
         - name: coredns

resources/coredns/service-account.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: calico-node
+  name: coredns
   namespace: kube-system

resources/coredns/service.yaml

@@ -8,7 +8,6 @@ metadata:
     prometheus.io/port: "9153"
   labels:
     k8s-app: coredns
-    kubernetes.io/cluster-service: "true"
     kubernetes.io/name: "CoreDNS"
 spec:
   selector:

resources/flannel/config.yaml

@@ -32,6 +32,6 @@ data:
       "Network": "${pod_cidr}",
       "Backend": {
         "Type": "vxlan",
-        "Port": 4789
+        "Port": 8472
       }
     }

resources/flannel/daemonset.yaml

@@ -17,17 +17,37 @@ spec:
     metadata:
       labels:
         k8s-app: flannel
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       hostNetwork: true
       priorityClassName: system-node-critical
       serviceAccountName: flannel
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+        - name: install-cni
+          image: ${flannel_cni_image}
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: flannel-config
+                  key: cni-conf.json
+          volumeMounts:
+            - name: cni-bin-dir
+              mountPath: /host/opt/cni/bin/
+            - name: cni-conf-dir
+              mountPath: /host/etc/cni/net.d
       containers:
         - name: flannel
           image: ${flannel_image}
@@ -55,20 +75,8 @@ spec:
               mountPath: /etc/kube-flannel/
             - name: run-flannel
               mountPath: /run/flannel
-        - name: install-cni
-          image: ${flannel_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: flannel-config
-                  key: cni-conf.json
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin/
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
+            - name: xtables-lock
+              mountPath: /run/xtables.lock
       volumes:
         - name: flannel-config
           configMap:
@@ -82,4 +90,10 @@ spec:
             path: /opt/cni/bin
         - name: cni-conf-dir
           hostPath:
-            path: /etc/kubernetes/cni/net.d
+            type: DirectoryOrCreate
+            path: /etc/cni/net.d
+        # Access iptables concurrently
+        - name: xtables-lock
+          hostPath:
+            type: FileOrCreate
+            path: /run/xtables.lock

resources/kube-proxy/kube-proxy.yaml

@@ -20,32 +20,42 @@ spec:
       labels:
         tier: node
         k8s-app: kube-proxy
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
     spec:
       hostNetwork: true
       priorityClassName: system-node-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
       serviceAccountName: kube-proxy
       tolerations:
-      - effect: NoSchedule
+      - key: node-role.kubernetes.io/controller
         operator: Exists
-      - effect: NoExecute
+      - key: node.kubernetes.io/not-ready
         operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
       containers:
       - name: kube-proxy
-        image: ${hyperkube_image}
+        image: ${kube_proxy_image}
         command:
-        - ./hyperkube
-        - proxy
+        - kube-proxy
         - --cluster-cidr=${pod_cidr}
         - --hostname-override=$(NODE_NAME)
         - --kubeconfig=/etc/kubernetes/kubeconfig
-        - --proxy-mode=iptables
+        - --metrics-bind-address=0.0.0.0
+        - --proxy-mode=ipvs
         env:
           - name: NODE_NAME
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
+        ports:
+        - name: metrics
+          containerPort: 10249
+        - name: health
+          containerPort: 10256
         livenessProbe:
           httpGet:
             path: /healthz
@@ -61,9 +71,14 @@ spec:
         - name: lib-modules
           mountPath: /lib/modules
           readOnly: true
-        - name: ssl-certs-host
+        - name: etc-ssl
           mountPath: /etc/ssl/certs
           readOnly: true
+        - name: etc-pki
+          mountPath: /etc/pki
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
       volumes:
       - name: kubeconfig
         configMap:
@@ -71,6 +86,14 @@ spec:
       - name: lib-modules
         hostPath:
           path: /lib/modules
-      - name: ssl-certs-host
+      - name: etc-ssl
         hostPath:
-          path: ${trusted_certs_dir}
+          path: /etc/ssl/certs
+      - name: etc-pki
+        hostPath:
+          path: /etc/pki
+      # Access iptables concurrently
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock

resources/kube-router/cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-router
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: kube-router
-subjects:
-  - kind: ServiceAccount
-    name: kube-router
-    namespace: kube-system

resources/kube-router/cluster-role.yaml

@@ -1,33 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
-kind: ClusterRole
-metadata:
-  name: kube-router
-rules:
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces
-      - pods
-      - services
-      - nodes
-      - endpoints
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-      - "networking.k8s.io"
-    resources:
-      - networkpolicies
-    verbs:
-      - list
-      - get
-      - watch
-  - apiGroups:
-      - extensions
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch

resources/kube-router/config.yaml

@@ -1,30 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-router-config
-  namespace: kube-system
-data:
-  cni-conf.json: |
-    {
-      "name": "pod-network",
-      "cniVersion": "0.3.1",
-      "plugins":[
-        {
-          "name": "kube-router",
-          "type": "bridge",
-          "bridge": "kube-bridge",
-          "isDefaultGateway": true,
-          "mtu": ${network_mtu},
-          "ipam": {
-            "type": "host-local"
-          }
-        },
-        {
-          "type": "portmap",
-          "snat": true,
-          "capabilities": {
-            "portMappings": true
-          }
-        }
-      ]
-    }

resources/kube-router/daemonset.yaml

@@ -1,90 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-router
-  namespace: kube-system
-  labels:
-    k8s-app: kube-router
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kube-router
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-router
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      priorityClassName: system-node-critical
-      serviceAccountName: kube-router
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-      containers:
-        - name: kube-router
-          image: ${kube_router_image}
-          args:
-            - --kubeconfig=/etc/kubernetes/kubeconfig
-            - --run-router=true
-            - --run-firewall=true
-            - --run-service-proxy=false
-            - --v=5
-          env:
-            - name: NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: KUBE_ROUTER_CNI_CONF_FILE
-              value: /etc/cni/net.d/10-kuberouter.conflist
-          securityContext:
-            privileged: true
-          volumeMounts:
-            - name: lib-modules
-              mountPath: /lib/modules
-              readOnly: true
-            - name: cni-conf-dir
-              mountPath: /etc/cni/net.d
-            - name: kubeconfig
-              mountPath: /etc/kubernetes
-              readOnly: true
-        - name: install-cni
-          image: ${flannel_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_OLD_NAME
-              value: 10-flannel.conflist
-            - name: CNI_CONF_NAME
-              value: 10-kuberouter.conflist
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: kube-router-config
-                  key: cni-conf.json
-          volumeMounts:
-            - name: cni-bin-dir
-              mountPath: /host/opt/cni/bin
-            - name: cni-conf-dir
-              mountPath: /host/etc/cni/net.d
-      volumes:
-        # Used by kube-router
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: kubeconfig
-          configMap:
-            name: kubeconfig-in-cluster
-        # Used by install-cni
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-conf-dir
-          hostPath:
-            path: /etc/kubernetes/cni/net.d

resources/kube-router/service-account.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: kube-router
-  namespace: kube-system

resources/kubeconfig-admin

@@ -1,18 +1,18 @@
 apiVersion: v1
 kind: Config
 clusters:
-- name: ${name}-cluster
+- name: ${name}
   cluster:
     server: ${server}
     certificate-authority-data: ${ca_cert}
 users:
-- name: ${name}-user
+- name: ${name}
   user:
     client-certificate-data: ${kubelet_cert}
     client-key-data: ${kubelet_key}
-current-context: ${name}-context
+current-context: ${name}
 contexts:
-- name: ${name}-context
+- name: ${name}
   context:
-    cluster: ${name}-cluster
-    user: ${name}-user
+    cluster: ${name}
+    user: ${name}

resources/kubeconfig-bootstrap

@@ -8,8 +8,7 @@ clusters:
 users:
 - name: kubelet
   user:
-    client-certificate-data: ${kubelet_cert}
-    client-key-data: ${kubelet_key}
+    token: ${token_id}.${token_secret}
 contexts:
 - context:
     cluster: local

resources/manifests/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Bind system:bootstrappers to ClusterRole for node bootstrap
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-bootstrapper
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve new CSRs from "system:bootstrappers" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-new
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve renewal CSRs from "system:nodes" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-renew
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/bootstrap-token.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>"
+  name: bootstrap-token-${token_id}
+  namespace: kube-system
+stringData:
+  description: "Typhoon generated bootstrap token"
+  token-id: ${token_id}
+  token-secret: ${token_secret}
+  usage-bootstrap-authentication: "true"

resources/manifests/coredns/service-account.yaml

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: coredns
-  namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"

resources/manifests/in-cluster.yaml

@@ -0,0 +1,10 @@
+# in-cluster ConfigMap is for control plane components that must reach
+# kube-apiserver before service IPs are available (e.g. 10.3.0.1)
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: in-cluster
+  namespace: kube-system
+data:
+  apiserver-host: ${apiserver_host}
+  apiserver-port: "${apiserver_port}"

resources/manifests/kube-apiserver-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-apiserver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: kube-apiserver
-  namespace: kube-system

resources/manifests/kube-apiserver-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-apiserver

resources/manifests/kube-apiserver-secret.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-type: Opaque
-data:
-  apiserver.key: ${apiserver_key}
-  apiserver.crt: ${apiserver_cert}
-  service-account.pub: ${serviceaccount_pub}
-  ca.crt: ${ca_cert}
-  etcd-client-ca.crt: ${etcd_ca_cert}
-  etcd-client.crt: ${etcd_client_cert}
-  etcd-client.key: ${etcd_client_key}
-  aggregation-ca.crt: ${aggregation_ca_cert}
-  aggregation-client.crt: ${aggregation_client_cert}
-  aggregation-client.key: ${aggregation_client_key}
-

resources/manifests/kube-apiserver.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-apiserver
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-apiserver
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-apiserver
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-apiserver
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-apiserver
-        image: ${hyperkube_image}
-        command:
-        - /hyperkube
-        - apiserver
-        - --advertise-address=$(POD_IP)
-        - --allow-privileged=true
-        - --anonymous-auth=false
-        - --authorization-mode=RBAC
-        - --bind-address=0.0.0.0
-        - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --cloud-provider=${cloud_provider}
-        - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority
-        - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-        - --etcd-servers=${etcd_servers}
-        - --insecure-port=0
-        - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-        - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
-        - --secure-port=${apiserver_port}
-        - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
-        - --service-cluster-ip-range=${service_cidr}
-        - --storage-backend=etcd3
-        - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-        - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
-        env:
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        volumeMounts:
-        - name: secrets
-          mountPath: /etc/kubernetes/secrets
-          readOnly: true
-        - name: ssl-certs-host
-          mountPath: /etc/ssl/certs
-          readOnly: true
-      volumes:
-      - name: secrets
-        secret:
-          secretName: kube-apiserver
-      - name: ssl-certs-host
-        hostPath:
-          path: ${trusted_certs_dir}

resources/manifests/kube-controller-manager-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager

resources/manifests/kube-controller-manager-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-controller-manager
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-controller-manager
-subjects:
-- kind: ServiceAccount
-  name: kube-controller-manager
-  namespace: kube-system

resources/manifests/kube-controller-manager-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-controller-manager

resources/manifests/kube-controller-manager-secret.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-type: Opaque
-data:
-  service-account.key: ${serviceaccount_key}
-  ca.crt: ${ca_cert}
-  ca.key: ${ca_key}
-

resources/manifests/kube-controller-manager.yaml

@@ -1,96 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-controller-manager
-spec:
-  replicas: ${control_plane_replicas}
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-controller-manager
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-controller-manager
-              topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      serviceAccountName: kube-controller-manager
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-controller-manager
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - controller-manager
-        - --use-service-account-credentials
-        - --allocate-node-cidrs=true
-        - --cloud-provider=${cloud_provider}
-        - --cluster-cidr=${pod_cidr}
-        - --service-cluster-ip-range=${service_cidr}
-        - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
-        - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
-        - --configure-cloud-routes=false
-        - --leader-elect=true
-        - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        - --pod-eviction-timeout=1m
-        - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
-        livenessProbe:
-          httpGet:
-            scheme: HTTPS
-            path: /healthz
-            port: 10257
-          initialDelaySeconds: 15
-          timeoutSeconds: 15
-        volumeMounts:
-        - name: secrets
-          mountPath: /etc/kubernetes/secrets
-          readOnly: true
-        - name: volumeplugins
-          mountPath: /var/lib/kubelet/volumeplugins
-          readOnly: true          
-        - name: ssl-host
-          mountPath: /etc/ssl/certs
-          readOnly: true
-      volumes:
-      - name: secrets
-        secret:
-          secretName: kube-controller-manager
-      - name: ssl-host
-        hostPath:
-          path: ${trusted_certs_dir}
-      - name: volumeplugins
-        hostPath:
-          path: /var/lib/kubelet/volumeplugins          
-      dnsPolicy: Default # Don't use cluster DNS.

resources/manifests/kube-scheduler-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler

resources/manifests/kube-scheduler-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-scheduler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-scheduler
-subjects:
-- kind: ServiceAccount
-  name: kube-scheduler
-  namespace: kube-system

resources/manifests/kube-scheduler-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-scheduler

resources/manifests/kube-scheduler-volume-scheduler-role-binding.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: volume-scheduler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:volume-scheduler
-subjects:
-- kind: ServiceAccount
-  name: kube-scheduler
-  namespace: kube-system
-

resources/manifests/kube-scheduler.yaml

@@ -1,63 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-scheduler
-spec:
-  replicas: ${control_plane_replicas}
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-scheduler
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-scheduler
-              topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      serviceAccountName: kube-scheduler
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-scheduler
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - scheduler
-        - --leader-elect=true
-        livenessProbe:
-          httpGet:
-            scheme: HTTPS
-            path: /healthz
-            port: 10259
-          initialDelaySeconds: 15
-          timeoutSeconds: 15

resources/manifests/kubelet-delete-cluster-role-binding.yaml

@@ -7,6 +7,6 @@ roleRef:
   kind: ClusterRole
   name: kubelet-delete
 subjects:
-- kind: Group
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
   name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/manifests/kubelet-delete-cluster-role.yaml

@@ -8,3 +8,16 @@ rules:
       - nodes
     verbs:
       - delete
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+    verbs:
+      - get
+      - list
+  - apiGroups: [""]
+    resources:
+      - pods/eviction
+    verbs:
+      - create

resources/manifests/kubelet-nodes-cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system-nodes
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node
-subjects:
-- kind: Group
-  name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/manifests/pod-checkpointer-cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: pod-checkpointer
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: pod-checkpointer
-subjects:
-- kind: ServiceAccount
-  name: pod-checkpointer
-  namespace: kube-system

resources/manifests/pod-checkpointer-cluster-role.yaml

@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: pod-checkpointer
-rules:
-  - apiGroups: [""]
-    resources:
-      - nodes
-      - nodes/proxy
-    verbs:
-      - get

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: pod-checkpointer
-subjects:
-- kind: ServiceAccount
-  name: pod-checkpointer
-  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-rules:
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["pods"]
-  verbs: ["get", "watch", "list"]
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["secrets", "configmaps"]
-  verbs: ["get"]

resources/manifests/pod-checkpointer-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: pod-checkpointer

resources/manifests/pod-checkpointer.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: pod-checkpointer
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: pod-checkpointer
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: pod-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-node-critical
-      serviceAccountName: pod-checkpointer
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: pod-checkpointer
-        image: ${pod_checkpointer_image}
-        command:
-        - /checkpoint
-        - --lock-file=/var/run/lock/pod-checkpointer.lock
-        - --kubeconfig=/etc/checkpointer/kubeconfig
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /etc/checkpointer
-        - name: etc-kubernetes
-          mountPath: /etc/kubernetes
-        - name: var-run
-          mountPath: /var/run
-      volumes:
-      - name: kubeconfig
-        configMap:
-          name: kubeconfig-in-cluster
-      - name: etc-kubernetes
-        hostPath:
-          path: /etc/kubernetes
-      - name: var-run
-        hostPath:
-          path: /var/run

resources/static-manifests/kube-apiserver.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-apiserver
+  namespace: kube-system
+  labels:
+    k8s-app: kube-apiserver
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-apiserver
+    image: ${kube_apiserver_image}
+    command:
+    - kube-apiserver
+    - --advertise-address=$(POD_IP)
+    - --allow-privileged=true
+    - --anonymous-auth=false
+    - --authorization-mode=Node,RBAC
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
+    - --enable-admission-plugins=NodeRestriction
+    - --enable-bootstrap-token-auth=true
+    - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.crt
+    - --etcd-certfile=/etc/kubernetes/pki/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/pki/etcd-client.key
+    - --etcd-servers=${etcd_servers}
+    - --feature-gates=MutatingAdmissionPolicy=true
+    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.crt
+    - --kubelet-client-key=/etc/kubernetes/pki/apiserver.key
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
+    - --runtime-config=admissionregistration.k8s.io/v1beta1=true,admissionregistration.k8s.io/v1alpha1=true
+    - --secure-port=6443
+    - --service-account-issuer=${service_account_issuer}
+    - --service-account-jwks-uri=${service_account_issuer}/openid/v1/jwks
+    - --service-account-key-file=/etc/kubernetes/pki/service-account.pub
+    - --service-account-signing-key-file=/etc/kubernetes/pki/service-account.key
+    - --service-cluster-ip-range=${service_cidr}
+    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
+    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    resources:
+      requests:
+        cpu: 150m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl/certs
+      readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki

resources/static-manifests/kube-controller-manager.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: kube-controller-manager
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-controller-manager
+    image: ${kube_controller_manager_image}
+    command:
+    - kube-controller-manager
+    - --authentication-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --allocate-node-cidrs=true
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
+    - --cluster-cidr=${pod_cidr}
+    - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
+    - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
+    - --cluster-signing-duration=72h
+    - --controllers=*,tokencleaner
+    - --configure-cloud-routes=false
+    - --kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --leader-elect=true
+    - --root-ca-file=/etc/kubernetes/pki/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/pki/service-account.key
+    - --service-cluster-ip-range=${service_cidr}
+    - --use-service-account-credentials=true
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10257
+      initialDelaySeconds: 25
+      timeoutSeconds: 15
+      failureThreshold: 8
+    resources:
+      requests:
+        cpu: 150m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl/certs
+      readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
+    - name: flex
+      mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki
+  - name: flex
+    hostPath:
+      type: DirectoryOrCreate
+      path: /var/lib/kubelet/volumeplugins

resources/static-manifests/kube-scheduler.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-scheduler
+  namespace: kube-system
+  labels:
+    k8s-app: kube-scheduler
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-scheduler
+    image: ${kube_scheduler_image}
+    command:
+    - kube-scheduler
+    - --authentication-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --leader-elect=true
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10259
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    resources:
+      requests:
+        cpu: 100m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki/scheduler.conf
+      readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki/scheduler.conf

terraform.tfvars.example

@@ -1,5 +1,4 @@
 cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
-asset_dir = "/home/core/mycluster"
 networking = "flannel"

tls-aggregation.tf

@@ -1,27 +1,26 @@
-# NOTE: Across this module, the following workaround is used:
-# `"${var.some_var == "condition" ? join(" ", tls_private_key.aggregation-ca.*.private_key_pem) : ""}"`
-# Due to https://github.com/hashicorp/hil/issues/50, both sides of conditions
-# are evaluated, until one of them is discarded. When a `count` is used resources
-# can be referenced as lists with the `.*` notation, and arrays are allowed to be
-# empty. The `join()` interpolation function is then used to cast them back to
-# a string. Since `count` can only be 0 or 1, the returned value is either empty
-# (and discarded anyways) or the desired value.
+locals {
+  # Kubernetes Aggregation TLS assets map
+  aggregation_tls = var.enable_aggregation ? {
+    "tls/k8s/aggregation-ca.crt"     = tls_self_signed_cert.aggregation-ca[0].cert_pem,
+    "tls/k8s/aggregation-client.crt" = tls_locally_signed_cert.aggregation-client[0].cert_pem,
+    "tls/k8s/aggregation-client.key" = tls_private_key.aggregation-client[0].private_key_pem,
+  } : {}
+}
 
 # Kubernetes Aggregation CA (i.e. front-proxy-ca)
 # Files: tls/{aggregation-ca.crt,aggregation-ca.key}
 
 resource "tls_private_key" "aggregation-ca" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_self_signed_cert" "aggregation-ca" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = "${tls_private_key.aggregation-ca.algorithm}"
-  private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
+  private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
 
   subject {
     common_name = "kubernetes-front-proxy-ca"
@@ -37,35 +36,20 @@ resource "tls_self_signed_cert" "aggregation-ca" {
   ]
 }
 
-resource "local_file" "aggregation-ca-key" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
-
-  content  = "${tls_private_key.aggregation-ca.private_key_pem}"
-  filename = "${var.asset_dir}/tls/aggregation-ca.key"
-}
-
-resource "local_file" "aggregation-ca-crt" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
-
-  content  = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/aggregation-ca.crt"
-}
-
 # Kubernetes apiserver (i.e. front-proxy-client)
 # Files: tls/{aggregation-client.crt,aggregation-client.key}
 
 resource "tls_private_key" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = "${tls_private_key.aggregation-client.algorithm}"
-  private_key_pem = "${tls_private_key.aggregation-client.private_key_pem}"
+  private_key_pem = tls_private_key.aggregation-client[0].private_key_pem
 
   subject {
     common_name = "kube-apiserver"
@@ -73,13 +57,12 @@ resource "tls_cert_request" "aggregation-client" {
 }
 
 resource "tls_locally_signed_cert" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  cert_request_pem = "${tls_cert_request.aggregation-client.cert_request_pem}"
+  cert_request_pem = tls_cert_request.aggregation-client[0].cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.aggregation-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
+  ca_private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.aggregation-ca[0].cert_pem
 
   validity_period_hours = 8760
 
@@ -90,16 +73,3 @@ resource "tls_locally_signed_cert" "aggregation-client" {
   ]
 }
 
-resource "local_file" "aggregation-client-key" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
-
-  content  = "${tls_private_key.aggregation-client.private_key_pem}"
-  filename = "${var.asset_dir}/tls/aggregation-client.key"
-}
-
-resource "local_file" "aggregation-client-crt" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
-
-  content  = "${tls_locally_signed_cert.aggregation-client.cert_pem}"
-  filename = "${var.asset_dir}/tls/aggregation-client.crt"
-}

tls-etcd.tf

@@ -1,70 +1,19 @@
-# etcd-ca.crt
-resource "local_file" "etcd_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-ca.crt"
+locals {
+  # etcd TLS assets map
+  etcd_tls = {
+    "tls/etcd/etcd-client-ca.crt" = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/etcd-client.crt"    = tls_locally_signed_cert.client.cert_pem,
+    "tls/etcd/etcd-client.key"    = tls_private_key.client.private_key_pem
+    "tls/etcd/server-ca.crt"      = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/server.crt"         = tls_locally_signed_cert.server.cert_pem
+    "tls/etcd/server.key"         = tls_private_key.server.private_key_pem
+    "tls/etcd/peer-ca.crt"        = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/peer.crt"           = tls_locally_signed_cert.peer.cert_pem
+    "tls/etcd/peer.key"           = tls_private_key.peer.private_key_pem
+  }
 }
 
-# etcd-ca.key
-resource "local_file" "etcd_ca_key" {
-  content  = "${tls_private_key.etcd-ca.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd-ca.key"
-}
-
-# etcd-client-ca.crt
-resource "local_file" "etcd_client_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
-}
-
-# etcd-client.crt
-resource "local_file" "etcd_client_crt" {
-  content  = "${tls_locally_signed_cert.client.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client.crt"
-}
-
-# etcd-client.key
-resource "local_file" "etcd_client_key" {
-  content  = "${tls_private_key.client.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client.key"
-}
-
-# server-ca.crt
-resource "local_file" "etcd_server_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
-}
-
-# server.crt
-resource "local_file" "etcd_server_crt" {
-  content  = "${tls_locally_signed_cert.server.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server.crt"
-}
-
-# server.key
-resource "local_file" "etcd_server_key" {
-  content  = "${tls_private_key.server.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server.key"
-}
-
-# peer-ca.crt
-resource "local_file" "etcd_peer_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
-}
-
-# peer.crt
-resource "local_file" "etcd_peer_crt" {
-  content  = "${tls_locally_signed_cert.peer.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer.crt"
-}
-
-# peer.key
-resource "local_file" "etcd_peer_key" {
-  content  = "${tls_private_key.peer.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer.key"
-}
-
-# certificates and keys
+# etcd CA
 
 resource "tls_private_key" "etcd-ca" {
   algorithm = "RSA"
@@ -72,8 +21,7 @@ resource "tls_private_key" "etcd-ca" {
 }
 
 resource "tls_self_signed_cert" "etcd-ca" {
-  key_algorithm   = "${tls_private_key.etcd-ca.algorithm}"
-  private_key_pem = "${tls_private_key.etcd-ca.private_key_pem}"
+  private_key_pem = tls_private_key.etcd-ca.private_key_pem
 
   subject {
     common_name  = "etcd-ca"
@@ -90,16 +38,15 @@ resource "tls_self_signed_cert" "etcd-ca" {
   ]
 }
 
-# client certs are used for client (apiserver, locksmith, etcd-operator)
-# to etcd communication
+# etcd Client (apiserver to etcd communication)
+
 resource "tls_private_key" "client" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "client" {
-  key_algorithm   = "${tls_private_key.client.algorithm}"
-  private_key_pem = "${tls_private_key.client.private_key_pem}"
+  private_key_pem = tls_private_key.client.private_key_pem
 
   subject {
     common_name  = "etcd-client"
@@ -110,19 +57,14 @@ resource "tls_cert_request" "client" {
     "127.0.0.1",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "client" {
-  cert_request_pem = "${tls_cert_request.client.cert_request_pem}"
+  cert_request_pem = tls_cert_request.client.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -134,14 +76,15 @@ resource "tls_locally_signed_cert" "client" {
   ]
 }
 
+# etcd Server
+
 resource "tls_private_key" "server" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "server" {
-  key_algorithm   = "${tls_private_key.server.algorithm}"
-  private_key_pem = "${tls_private_key.server.private_key_pem}"
+  private_key_pem = tls_private_key.server.private_key_pem
 
   subject {
     common_name  = "etcd-server"
@@ -152,19 +95,14 @@ resource "tls_cert_request" "server" {
     "127.0.0.1",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "server" {
-  cert_request_pem = "${tls_cert_request.server.cert_request_pem}"
+  cert_request_pem = tls_cert_request.server.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -176,29 +114,29 @@ resource "tls_locally_signed_cert" "server" {
   ]
 }
 
+# etcd Peer
+
 resource "tls_private_key" "peer" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "peer" {
-  key_algorithm   = "${tls_private_key.peer.algorithm}"
-  private_key_pem = "${tls_private_key.peer.private_key_pem}"
+  private_key_pem = tls_private_key.peer.private_key_pem
 
   subject {
     common_name  = "etcd-peer"
     organization = "etcd"
   }
 
-  dns_names = ["${var.etcd_servers}"]
+  dns_names = var.etcd_servers
 }
 
 resource "tls_locally_signed_cert" "peer" {
-  cert_request_pem = "${tls_cert_request.peer.cert_request_pem}"
+  cert_request_pem = tls_cert_request.peer.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -209,3 +147,4 @@ resource "tls_locally_signed_cert" "peer" {
     "client_auth",
   ]
 }
+

tls-k8s.tf

@@ -1,3 +1,15 @@
+locals {
+  # Kubernetes TLS assets map
+  kubernetes_tls = {
+    "tls/k8s/ca.crt"              = tls_self_signed_cert.kube-ca.cert_pem,
+    "tls/k8s/ca.key"              = tls_private_key.kube-ca.private_key_pem,
+    "tls/k8s/apiserver.crt"       = tls_locally_signed_cert.apiserver.cert_pem,
+    "tls/k8s/apiserver.key"       = tls_private_key.apiserver.private_key_pem,
+    "tls/k8s/service-account.pub" = tls_private_key.service-account.public_key_pem
+    "tls/k8s/service-account.key" = tls_private_key.service-account.private_key_pem
+  }
+}
+
 # Kubernetes CA (tls/{ca.crt,ca.key})
 
 resource "tls_private_key" "kube-ca" {
@@ -6,12 +18,11 @@ resource "tls_private_key" "kube-ca" {
 }
 
 resource "tls_self_signed_cert" "kube-ca" {
-  key_algorithm   = "${tls_private_key.kube-ca.algorithm}"
-  private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
+  private_key_pem = tls_private_key.kube-ca.private_key_pem
 
   subject {
     common_name  = "kubernetes-ca"
-    organization = "bootkube"
+    organization = "typhoon"
   }
 
   is_ca_certificate     = true
@@ -24,16 +35,6 @@ resource "tls_self_signed_cert" "kube-ca" {
   ]
 }
 
-resource "local_file" "kube-ca-key" {
-  content  = "${tls_private_key.kube-ca.private_key_pem}"
-  filename = "${var.asset_dir}/tls/ca.key"
-}
-
-resource "local_file" "kube-ca-crt" {
-  content  = "${tls_self_signed_cert.kube-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/ca.crt"
-}
-
 # Kubernetes API Server (tls/{apiserver.key,apiserver.crt})
 
 resource "tls_private_key" "apiserver" {
@@ -42,33 +43,31 @@ resource "tls_private_key" "apiserver" {
 }
 
 resource "tls_cert_request" "apiserver" {
-  key_algorithm   = "${tls_private_key.apiserver.algorithm}"
-  private_key_pem = "${tls_private_key.apiserver.private_key_pem}"
+  private_key_pem = tls_private_key.apiserver.private_key_pem
 
   subject {
     common_name  = "kube-apiserver"
     organization = "system:masters"
   }
 
-  dns_names = [
-    "${var.api_servers}",
+  dns_names = flatten([
+    var.api_servers,
     "kubernetes",
     "kubernetes.default",
     "kubernetes.default.svc",
     "kubernetes.default.svc.${var.cluster_domain_suffix}",
-  ]
+  ])
 
   ip_addresses = [
-    "${cidrhost(var.service_cidr, 1)}",
+    cidrhost(var.service_cidr, 1),
   ]
 }
 
 resource "tls_locally_signed_cert" "apiserver" {
-  cert_request_pem = "${tls_cert_request.apiserver.cert_request_pem}"
+  cert_request_pem = tls_cert_request.apiserver.cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -80,14 +79,64 @@ resource "tls_locally_signed_cert" "apiserver" {
   ]
 }
 
-resource "local_file" "apiserver-key" {
-  content  = "${tls_private_key.apiserver.private_key_pem}"
-  filename = "${var.asset_dir}/tls/apiserver.key"
+# kube-controller-manager
+
+resource "tls_private_key" "controller-manager" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
 }
 
-resource "local_file" "apiserver-crt" {
-  content  = "${tls_locally_signed_cert.apiserver.cert_pem}"
-  filename = "${var.asset_dir}/tls/apiserver.crt"
+resource "tls_cert_request" "controller-manager" {
+  private_key_pem = tls_private_key.controller-manager.private_key_pem
+
+  subject {
+    common_name = "system:kube-controller-manager"
+  }
+}
+
+resource "tls_locally_signed_cert" "controller-manager" {
+  cert_request_pem = tls_cert_request.controller-manager.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+# kube-scheduler
+
+resource "tls_private_key" "scheduler" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "scheduler" {
+  private_key_pem = tls_private_key.scheduler.private_key_pem
+
+  subject {
+    common_name = "system:kube-scheduler"
+  }
+}
+
+resource "tls_locally_signed_cert" "scheduler" {
+  cert_request_pem = tls_cert_request.scheduler.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
 }
 
 # Kubernetes Admin (tls/{admin.key,admin.crt})
@@ -98,8 +147,7 @@ resource "tls_private_key" "admin" {
 }
 
 resource "tls_cert_request" "admin" {
-  key_algorithm   = "${tls_private_key.admin.algorithm}"
-  private_key_pem = "${tls_private_key.admin.private_key_pem}"
+  private_key_pem = tls_private_key.admin.private_key_pem
 
   subject {
     common_name  = "kubernetes-admin"
@@ -108,11 +156,10 @@ resource "tls_cert_request" "admin" {
 }
 
 resource "tls_locally_signed_cert" "admin" {
-  cert_request_pem = "${tls_cert_request.admin.cert_request_pem}"
+  cert_request_pem = tls_cert_request.admin.cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -123,16 +170,6 @@ resource "tls_locally_signed_cert" "admin" {
   ]
 }
 
-resource "local_file" "admin-key" {
-  content  = "${tls_private_key.admin.private_key_pem}"
-  filename = "${var.asset_dir}/tls/admin.key"
-}
-
-resource "local_file" "admin-crt" {
-  content  = "${tls_locally_signed_cert.admin.cert_pem}"
-  filename = "${var.asset_dir}/tls/admin.crt"
-}
-
 # Kubernete's Service Account (tls/{service-account.key,service-account.pub})
 
 resource "tls_private_key" "service-account" {
@@ -140,56 +177,3 @@ resource "tls_private_key" "service-account" {
   rsa_bits  = "2048"
 }
 
-resource "local_file" "service-account-key" {
-  content  = "${tls_private_key.service-account.private_key_pem}"
-  filename = "${var.asset_dir}/tls/service-account.key"
-}
-
-resource "local_file" "service-account-crt" {
-  content  = "${tls_private_key.service-account.public_key_pem}"
-  filename = "${var.asset_dir}/tls/service-account.pub"
-}
-
-# Kubelet
-
-resource "tls_private_key" "kubelet" {
-  algorithm = "RSA"
-  rsa_bits  = "2048"
-}
-
-resource "tls_cert_request" "kubelet" {
-  key_algorithm   = "${tls_private_key.kubelet.algorithm}"
-  private_key_pem = "${tls_private_key.kubelet.private_key_pem}"
-
-  subject {
-    common_name  = "kubelet"
-    organization = "system:nodes"
-  }
-}
-
-resource "tls_locally_signed_cert" "kubelet" {
-  cert_request_pem = "${tls_cert_request.kubelet.cert_request_pem}"
-
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
-
-  validity_period_hours = 8760
-
-  allowed_uses = [
-    "key_encipherment",
-    "digital_signature",
-    "server_auth",
-    "client_auth",
-  ]
-}
-
-resource "local_file" "kubelet-key" {
-  content  = "${tls_private_key.kubelet.private_key_pem}"
-  filename = "${var.asset_dir}/tls/kubelet.key"
-}
-
-resource "local_file" "kubelet-crt" {
-  content  = "${tls_locally_signed_cert.kubelet.cert_pem}"
-  filename = "${var.asset_dir}/tls/kubelet.crt"
-}

variables.tf

@@ -1,113 +1,140 @@
 variable "cluster_name" {
+  type        = string
   description = "Cluster name"
-  type        = "string"
 }
 
 variable "api_servers" {
+  type        = list(string)
   description = "List of URLs used to reach kube-apiserver"
-  type        = "list"
 }
 
 variable "etcd_servers" {
+  type        = list(string)
   description = "List of URLs used to reach etcd servers."
-  type        = "list"
 }
 
-variable "asset_dir" {
-  description = "Path to a directory where generated assets should be placed (contains secrets)"
-  type        = "string"
-}
-
-variable "cloud_provider" {
-  description = "The provider for cloud services (empty string for no provider)"
-  type        = "string"
-  default     = ""
-}
+# optional
 
 variable "networking" {
-  description = "Choice of networking provider (flannel or calico or kube-router)"
-  type        = "string"
-  default     = "flannel"
-}
-
-variable "network_mtu" {
-  description = "CNI interface MTU (only applies to calico and kube-router)"
-  type        = "string"
-  default     = "1500"
-}
-
-variable "network_encapsulation" {
-  description = "Network encapsulation mode either ipip or vxlan (only applies to calico)"
-  type        = "string"
-  default     = "ipip"
-}
-
-variable "network_ip_autodetection_method" {
-  description = "Method to autodetect the host IPv4 address (only applies to calico)"
-  type        = "string"
-  default     = "first-found"
+  type        = string
+  description = "Choice of networking provider (flannel or cilium)"
+  default     = "cilium"
+  validation {
+    condition     = contains(["flannel", "cilium"], var.networking)
+    error_message = "networking can be flannel or cilium."
+  }
 }
 
 variable "pod_cidr" {
+  type        = string
   description = "CIDR IP range to assign Kubernetes pods"
-  type        = "string"
-  default     = "10.2.0.0/16"
+  default     = "10.20.0.0/14"
 }
 
 variable "service_cidr" {
+  type        = string
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
 The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
 EOD
-
-  type    = "string"
-  default = "10.3.0.0/24"
-}
-
-variable "cluster_domain_suffix" {
-  description = "Queries for domains with the suffix will be answered by kube-dns"
-  type        = "string"
-  default     = "cluster.local"
+  default     = "10.3.0.0/24"
 }
 
 variable "container_images" {
+  type        = map(string)
   description = "Container images to use"
-  type        = "map"
-
   default = {
-    calico           = "quay.io/calico/node:v3.7.2"
-    calico_cni       = "quay.io/calico/cni:v3.7.2"
-    flannel          = "quay.io/coreos/flannel:v0.11.0-amd64"
-    flannel_cni      = "quay.io/coreos/flannel-cni:v0.3.0"
-    kube_router      = "cloudnativelabs/kube-router:v0.3.1"
-    hyperkube        = "k8s.gcr.io/hyperkube:v1.14.3"
-    coredns          = "k8s.gcr.io/coredns:1.5.0"
-    pod_checkpointer = "quay.io/coreos/pod-checkpointer:83e25e5968391b9eb342042c435d1b3eeddb2be1"
+    cilium_agent            = "quay.io/cilium/cilium:v1.18.4"
+    cilium_operator         = "quay.io/cilium/operator-generic:v1.18.4"
+    coredns                 = "registry.k8s.io/coredns/coredns:v1.13.1"
+    flannel                 = "docker.io/flannel/flannel:v0.27.0"
+    flannel_cni             = "quay.io/poseidon/flannel-cni:v0.4.2"
+    kube_apiserver          = "registry.k8s.io/kube-apiserver:v1.34.2"
+    kube_controller_manager = "registry.k8s.io/kube-controller-manager:v1.34.2"
+    kube_scheduler          = "registry.k8s.io/kube-scheduler:v1.34.2"
+    kube_proxy              = "registry.k8s.io/kube-proxy:v1.34.2"
   }
 }
 
-variable "enable_reporting" {
-  type        = "string"
-  description = "Enable usage or analytics reporting to upstream component owners (Tigera: Calico)"
-  default     = "false"
-}
-
-variable "trusted_certs_dir" {
-  description = "Path to the directory on cluster nodes where trust TLS certs are kept"
-  type        = "string"
-  default     = "/usr/share/ca-certificates"
-}
-
 variable "enable_aggregation" {
-  description = "Enable the Kubernetes Aggregation Layer (defaults to false, recommended)"
-  type        = "string"
-  default     = "false"
+  type        = bool
+  description = "Enable the Kubernetes Aggregation Layer (defaults to true)"
+  default     = true
+}
+
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
 }
 
 # unofficial, temporary, may be removed without notice
 
-variable "apiserver_port" {
-  description = "kube-apiserver port"
-  type        = "string"
-  default     = "6443"
+variable "external_apiserver_port" {
+  type        = number
+  description = "External kube-apiserver port (e.g. 6443 to match internal kube-apiserver port)"
+  default     = 6443
+}
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by kube-dns"
+  default     = "cluster.local"
+}
+
+variable "components" {
+  description = "Configure pre-installed cluster components"
+  type = object({
+    enable = optional(bool, true)
+    coredns = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    kube_proxy = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    # CNI providers are enabled for pre-install by default, but only the
+    # provider matching var.networking is actually installed.
+    flannel = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    cilium = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+  })
+  default = {
+    enable     = true
+    coredns    = null
+    kube_proxy = null
+    flannel    = null
+    cilium     = null
+  }
+  # Set the variable value to the default value when the caller
+  # sets it to null.
+  nullable = false
+}
+
+variable "service_account_issuer" {
+  type        = string
+  description = "kube-apiserver service account token issuer (used as an identifier in 'iss' claims)"
+  default     = "https://kubernetes.default.svc.cluster.local"
 }

versions.tf

@@ -0,0 +1,9 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.13.0, < 2.0.0"
+  required_providers {
+    random = "~> 3.1"
+    tls    = "~> 4.0"
+  }
+}