Update kube-apiserver runtime-config flags

* MutatingAdmissionPolicy is available as a beta and alpha API

.github/dependabot.yaml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+    interval: "daily"

.github/workflows/test.yaml

@@ -0,0 +1,21 @@
+name: test
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+jobs:
+  terraform:
+    name: fmt
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v6
+
+      - name: terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.11.1
+
+      - name: fmt
+        run: terraform fmt -check -diff -recursive

README.md

@@ -1,70 +1,42 @@
-# terraform-render-bootkube
+# terraform-render-bootstrap
+[![Workflow](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml/badge.svg)](https://github.com/poseidon/terraform-render-bootstrap/actions/workflows/test.yaml?query=branch%3Amain)
+[![Sponsors](https://img.shields.io/github/sponsors/poseidon?logo=github)](https://github.com/sponsors/poseidon)
+[![Mastodon](https://img.shields.io/badge/follow-news-6364ff?logo=mastodon)](https://fosstodon.org/@typhoon)
 
-`terraform-render-bootkube` is a Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) assets for bootstrapping a Kubernetes cluster.
+`terraform-render-bootstrap` is a Terraform module that renders TLS certificates, static pods, and manifests for bootstrapping a Kubernetes cluster.
 
 ## Audience
 
-`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+`terraform-render-bootstrap` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootstrap module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
-Use the module to declare bootkube assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
+Use the module to declare bootstrap assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
 
 ```hcl
-module "bootkube" {
-  source = "git://https://github.com/poseidon/terraform-render-bootkube.git?ref=SHA"
+module "bootstrap" {
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=SHA"
 
   cluster_name = "example"
   api_servers = ["node1.example.com"]
   etcd_servers = ["node1.example.com"]
-  asset_dir = "/home/core/clusters/mycluster"
 }
 ```
 
-Generate the assets.
+Generate assets in Terraform state.
 
 ```sh
 terraform init
-terraform get --update
 terraform plan
 terraform apply
 ```
 
-Find bootkube assets rendered to the `asset_dir` path. That's it.
+To inspect and write assets locally (e.g. debugging) use the `assets_dist` Terraform output.
 
-### Comparison
-
-Render bootkube assets directly with bootkube v0.9.0.
-
-#### On-host etcd (recommended)
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
 ```
-
-Compare assets. The only diffs you should see are TLS credentials.
-
-```sh
-pushd /home/core/mycluster
-mv manifests-networking/* manifests
-popd
-diff -rw assets /home/core/mycluster
+resource local_file "assets" {
+  for_each = module.bootstrap.assets_dist
+  filename = "some-assets/${each.key}"
+  content = each.value
+}
 ```
-
-#### Self-hosted etcd (deprecated)
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd
-```
-
-Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment.
-
-```sh
-pushd /home/core/mycluster
-mv experimental/bootstrap-manifests/* boostrap-manifests
-mv experimental/manifests/* manifests
-mv manifests-networking/* manifests
-popd
-diff -rw assets /home/core/mycluster
-```
-

assets.tf

@@ -1,81 +0,0 @@
-# Self-hosted Kubernetes bootstrap-manifests
-resource "template_dir" "bootstrap-manifests" {
-  source_dir      = "${path.module}/resources/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/bootstrap-manifests"
-
-  vars {
-    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379,https://127.0.0.1:12379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-
-    cloud_provider = "${var.cloud_provider}"
-    pod_cidr       = "${var.pod_cidr}"
-    service_cidr   = "${var.service_cidr}"
-  }
-}
-
-# Self-hosted Kubernetes manifests
-resource "template_dir" "manifests" {
-  source_dir      = "${path.module}/resources/manifests"
-  destination_dir = "${var.asset_dir}/manifests"
-
-  vars {
-    hyperkube_image        = "${var.container_images["hyperkube"]}"
-    pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}"
-    kubedns_image          = "${var.container_images["kubedns"]}"
-    kubedns_dnsmasq_image  = "${var.container_images["kubedns_dnsmasq"]}"
-    kubedns_sidecar_image  = "${var.container_images["kubedns_sidecar"]}"
-
-    etcd_servers = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-
-    cloud_provider      = "${var.cloud_provider}"
-    pod_cidr            = "${var.pod_cidr}"
-    service_cidr        = "${var.service_cidr}"
-    kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
-
-    ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-    server       = "${format("https://%s:443", element(var.api_servers, 0))}"
-    apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
-    apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
-    serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
-    serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
-
-    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
-  }
-}
-
-# Generated kubeconfig
-resource "local_file" "kubeconfig" {
-  content  = "${data.template_file.kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/kubeconfig"
-}
-
-# Generated kubeconfig with user-context
-resource "local_file" "user-kubeconfig" {
-  content  = "${data.template_file.user-kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
-}
-
-data "template_file" "kubeconfig" {
-  template = "${file("${path.module}/resources/kubeconfig")}"
-
-  vars {
-    ca_cert      = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-    server       = "${format("https://%s:443", element(var.api_servers, 0))}"
-  }
-}
-
-data "template_file" "user-kubeconfig" {
-  template = "${file("${path.module}/resources/user-kubeconfig")}"
-
-  vars {
-    name         = "${var.cluster_name}"
-    ca_cert      = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-    server       = "${format("https://%s:443", element(var.api_servers, 0))}"
-  }
-}

auth.tf

@@ -0,0 +1,68 @@
+locals {
+  # component kubeconfigs assets map
+  auth_kubeconfigs = {
+    "auth/admin.conf"              = local.kubeconfig-admin,
+    "auth/controller-manager.conf" = local.kubeconfig-controller-manager
+    "auth/scheduler.conf"          = local.kubeconfig-scheduler
+  }
+}
+
+locals {
+  # Generated admin kubeconfig to bootstrap control plane
+  kubeconfig-admin = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-controller-manager = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.controller-manager.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.controller-manager.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kube-controller-manager kubeconfig
+  kubeconfig-scheduler = templatefile("${path.module}/resources/kubeconfig-admin",
+    {
+      name         = var.cluster_name
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      kubelet_cert = base64encode(tls_locally_signed_cert.scheduler.cert_pem)
+      kubelet_key  = base64encode(tls_private_key.scheduler.private_key_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    }
+  )
+
+  # Generated kubeconfig to bootstrap Kubelets
+  kubeconfig-bootstrap = templatefile("${path.module}/resources/kubeconfig-bootstrap",
+    {
+      ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+      server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+      token_id     = random_password.bootstrap-token-id.result
+      token_secret = random_password.bootstrap-token-secret.result
+    }
+  )
+}
+
+# Generate a cryptographically random token id (public)
+resource "random_password" "bootstrap-token-id" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret
+resource "random_password" "bootstrap-token-secret" {
+  length  = 16
+  upper   = false
+  special = false
+}
+

conditional.tf

@@ -1,74 +1,36 @@
-# Assets generated only when experimental self-hosted etcd is enabled
+# Assets generated only when certain options are chosen
 
-resource "template_dir" "flannel-manifests" {
-  count           = "${var.networking == "flannel" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/flannel"
-  destination_dir = "${var.asset_dir}/manifests-networking"
+locals {
+  # flannel manifests map
+  # { manifests-networking/manifest.yaml => content }
+  flannel_manifests = {
+    for name in fileset("${path.module}/resources/flannel", "*.yaml") :
+    "manifests/network/${name}" => templatefile(
+      "${path.module}/resources/flannel/${name}",
+      {
+        flannel_image         = var.container_images["flannel"]
+        flannel_cni_image     = var.container_images["flannel_cni"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
+      }
+    )
+    if var.components.enable && var.components.flannel.enable && var.networking == "flannel"
+  }
 
-  vars {
-    flannel_image     = "${var.container_images["flannel"]}"
-    flannel_cni_image = "${var.container_images["flannel_cni"]}"
-
-    pod_cidr = "${var.pod_cidr}"
+  # cilium manifests map
+  # { manifests-networking/manifest.yaml => content }
+  cilium_manifests = {
+    for name in fileset("${path.module}/resources/cilium", "**/*.yaml") :
+    "manifests/network/${name}" => templatefile(
+      "${path.module}/resources/cilium/${name}",
+      {
+        cilium_agent_image    = var.container_images["cilium_agent"]
+        cilium_operator_image = var.container_images["cilium_operator"]
+        pod_cidr              = var.pod_cidr
+        daemonset_tolerations = var.daemonset_tolerations
+      }
+    )
+    if var.components.enable && var.components.cilium.enable && var.networking == "cilium"
   }
 }
 
-resource "template_dir" "calico-manifests" {
-  count           = "${var.networking == "calico" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/calico"
-  destination_dir = "${var.asset_dir}/manifests-networking"
-
-  vars {
-    calico_image     = "${var.container_images["calico"]}"
-    calico_cni_image = "${var.container_images["calico_cni"]}"
-
-    network_mtu = "${var.network_mtu}"
-    pod_cidr    = "${var.pod_cidr}"
-  }
-}
-
-# bootstrap-etcd.yaml pod bootstrap-manifest
-resource "template_dir" "experimental-bootstrap-manifests" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"
-
-  vars {
-    etcd_image                = "${var.container_images["etcd"]}"
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
-resource "template_dir" "etcd-subfolder" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/etcd"
-  destination_dir = "${var.asset_dir}/etcd"
-
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd-operator deployment and etcd-service manifests
-# etcd client, server, and peer tls secrets
-resource "template_dir" "experimental-manifests" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/manifests"
-  destination_dir = "${var.asset_dir}/experimental/manifests"
-
-  vars {
-    etcd_operator_image     = "${var.container_images["etcd_operator"]}"
-    etcd_checkpointer_image = "${var.container_images["etcd_checkpointer"]}"
-    etcd_service_ip         = "${cidrhost(var.service_cidr, 15)}"
-
-    # Self-hosted etcd TLS certs / keys
-    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
-    etcd_server_cert = "${base64encode(tls_locally_signed_cert.server.cert_pem)}"
-    etcd_server_key  = "${base64encode(tls_private_key.server.private_key_pem)}"
-    etcd_peer_cert   = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
-    etcd_peer_key    = "${base64encode(tls_private_key.peer.private_key_pem)}"
-  }
-}

manifests.tf

@@ -0,0 +1,77 @@
+locals {
+  # Kubernetes static pod manifests map
+  # {static-manifests/manifest.yaml => content }
+  static_manifests = {
+    for name in fileset("${path.module}/resources/static-manifests", "*.yaml") :
+    "static-manifests/${name}" => templatefile(
+      "${path.module}/resources/static-manifests/${name}",
+      {
+        kube_apiserver_image          = var.container_images["kube_apiserver"]
+        kube_controller_manager_image = var.container_images["kube_controller_manager"]
+        kube_scheduler_image          = var.container_images["kube_scheduler"]
+
+        etcd_servers = join(",", formatlist("https://%s:2379", var.etcd_servers))
+        pod_cidr     = var.pod_cidr
+        service_cidr = var.service_cidr
+
+        service_account_issuer = var.service_account_issuer
+        aggregation_flags      = var.enable_aggregation ? indent(4, local.aggregation_flags) : ""
+      }
+    )
+  }
+
+  # Kubernetes control plane manifests map
+  # { manifests/manifest.yaml => content }
+  manifests = merge({
+    for name in fileset("${path.module}/resources/manifests", "**/*.yaml") :
+    "manifests/${name}" => templatefile(
+      "${path.module}/resources/manifests/${name}",
+      {
+        server         = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+        apiserver_host = var.api_servers[0]
+        apiserver_port = var.external_apiserver_port
+        token_id       = random_password.bootstrap-token-id.result
+        token_secret   = random_password.bootstrap-token-secret.result
+      }
+    )
+    },
+    # CoreDNS manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/coredns", "*.yaml") :
+      "manifests/coredns/${name}" => templatefile(
+        "${path.module}/resources/coredns/${name}",
+        {
+          coredns_image          = var.container_images["coredns"]
+          control_plane_replicas = max(2, length(var.etcd_servers))
+          cluster_domain_suffix  = var.cluster_domain_suffix
+          cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+        }
+      ) if var.components.enable && var.components.coredns.enable
+    },
+    # kube-proxy manifests (optional)
+    {
+      for name in fileset("${path.module}/resources/kube-proxy", "*.yaml") :
+      "manifests/kube-proxy/${name}" => templatefile(
+        "${path.module}/resources/kube-proxy/${name}",
+        {
+          kube_proxy_image      = var.container_images["kube_proxy"]
+          pod_cidr              = var.pod_cidr
+          daemonset_tolerations = var.daemonset_tolerations
+        }
+      ) if var.components.enable && var.components.kube_proxy.enable && var.networking != "cilium"
+    }
+  )
+}
+
+locals {
+  aggregation_flags = <<EOF
+
+- --proxy-client-cert-file=/etc/kubernetes/pki/aggregation-client.crt
+- --proxy-client-key-file=/etc/kubernetes/pki/aggregation-client.key
+- --requestheader-client-ca-file=/etc/kubernetes/pki/aggregation-ca.crt
+- --requestheader-extra-headers-prefix=X-Remote-Extra-
+- --requestheader-group-headers=X-Remote-Group
+- --requestheader-username-headers=X-Remote-User
+EOF
+}
+

outputs.tf

@@ -1,73 +1,76 @@
-output "id" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${local_file.kubeconfig.id}")}"
+
+output "cluster_dns_service_ip" {
+  value = cidrhost(var.service_cidr, 10)
 }
 
-output "content_hash" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
+// Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
+output "kubeconfig-kubelet" {
+  value     = local.kubeconfig-bootstrap
+  sensitive = true
 }
 
-output "kube_dns_service_ip" {
-  value = "${cidrhost(var.service_cidr, 10)}"
+// Generated kubeconfig for admins (i.e. human super-user)
+output "kubeconfig-admin" {
+  value     = local.kubeconfig-admin
+  sensitive = true
 }
 
-output "etcd_service_ip" {
-  value = "${cidrhost(var.service_cidr, 15)}"
-}
-
-output "kubeconfig" {
-  value = "${data.template_file.kubeconfig.rendered}"
-}
-
-output "user-kubeconfig" {
-  value = "${local_file.user-kubeconfig.filename}"
+# assets to distribute to controllers
+# { some/path => content }
+output "assets_dist" {
+  # combine maps of assets
+  value = merge(
+    local.auth_kubeconfigs,
+    local.etcd_tls,
+    local.kubernetes_tls,
+    local.aggregation_tls,
+    local.static_manifests,
+    local.manifests,
+    local.flannel_manifests,
+    local.cilium_manifests,
+  )
+  sensitive = true
 }
 
 # etcd TLS assets
 
 output "etcd_ca_cert" {
-  value = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  value     = tls_self_signed_cert.etcd-ca.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_cert" {
-  value = "${tls_locally_signed_cert.client.cert_pem}"
+  value     = tls_locally_signed_cert.client.cert_pem
+  sensitive = true
 }
 
 output "etcd_client_key" {
-  value = "${tls_private_key.client.private_key_pem}"
+  value     = tls_private_key.client.private_key_pem
+  sensitive = true
 }
 
 output "etcd_server_cert" {
-  value = "${tls_locally_signed_cert.server.cert_pem}"
+  value     = tls_locally_signed_cert.server.cert_pem
+  sensitive = true
 }
 
 output "etcd_server_key" {
-  value = "${tls_private_key.server.private_key_pem}"
+  value     = tls_private_key.server.private_key_pem
+  sensitive = true
 }
 
 output "etcd_peer_cert" {
-  value = "${tls_locally_signed_cert.peer.cert_pem}"
+  value     = tls_locally_signed_cert.peer.cert_pem
+  sensitive = true
 }
 
 output "etcd_peer_key" {
-  value = "${tls_private_key.peer.private_key_pem}"
+  value     = tls_private_key.peer.private_key_pem
+  sensitive = true
 }
 
-# Some platforms may need to reconstruct the kubeconfig directly in user-data.
-# That can't be done with the way template_file interpolates multi-line
-# contents so the raw components of the kubeconfig may be needed.
+# Kubernetes TLS assets
 
-output "ca_cert" {
-  value = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-}
-
-output "kubelet_cert" {
-  value = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-}
-
-output "kubelet_key" {
-  value = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-}
-
-output "server" {
-  value = "${format("https://%s:443", element(var.api_servers, 0))}"
+output "service_account_public_key" {
+  value = tls_private_key.service-account.public_key_pem
 }

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -1,60 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-apiserver
-  namespace: kube-system
-spec:
-  containers:
-  - name: kube-apiserver
-    image: ${hyperkube_image}
-    command:
-    - /hyperkube
-    - apiserver
-    - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
-    - --advertise-address=$(POD_IP)
-    - --allow-privileged=true
-    - --authorization-mode=RBAC
-    - --bind-address=0.0.0.0
-    - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-    - --etcd-quorum-read=true
-    - --etcd-servers=${etcd_servers}
-    - --insecure-port=0
-    - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-    - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-    - --secure-port=443
-    - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
-    - --service-cluster-ip-range=${service_cidr}
-    - --cloud-provider=${cloud_provider}
-    - --storage-backend=etcd3
-    - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-    - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
-    env:
-    - name: POD_IP
-      valueFrom:
-        fieldRef:
-          fieldPath: status.podIP
-    volumeMounts:
-    - mountPath: /etc/ssl/certs
-      name: ssl-certs-host
-      readOnly: true
-    - mountPath: /etc/kubernetes/secrets
-      name: secrets
-      readOnly: true
-    - mountPath: /var/lock
-      name: var-lock
-      readOnly: false
-  hostNetwork: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets
-  - name: ssl-certs-host
-    hostPath:
-      path: /usr/share/ca-certificates
-  - name: var-lock
-    hostPath:
-      path: /var/lock

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -1,35 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-controller-manager
-  namespace: kube-system
-spec:
-  containers:
-  - name: kube-controller-manager
-    image: ${hyperkube_image}
-    command:
-    - ./hyperkube
-    - controller-manager
-    - --allocate-node-cidrs=true
-    - --cluster-cidr=${pod_cidr}
-    - --cloud-provider=${cloud_provider}
-    - --configure-cloud-routes=false
-    - --kubeconfig=/etc/kubernetes/kubeconfig
-    - --leader-elect=true
-    - --root-ca-file=/etc/kubernetes/bootstrap-secrets/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/bootstrap-secrets/service-account.key
-    volumeMounts:
-    - name: kubernetes
-      mountPath: /etc/kubernetes
-      readOnly: true
-    - name: ssl-host
-      mountPath: /etc/ssl/certs
-      readOnly: true
-  hostNetwork: true
-  volumes:
-  - name: kubernetes
-    hostPath:
-      path: /etc/kubernetes
-  - name: ssl-host
-    hostPath:
-      path: /usr/share/ca-certificates

resources/bootstrap-manifests/bootstrap-scheduler.yaml

@@ -1,23 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-kube-scheduler
-  namespace: kube-system
-spec:
-  containers:
-  - name: kube-scheduler
-    image: ${hyperkube_image}
-    command:
-    - ./hyperkube
-    - scheduler
-    - --kubeconfig=/etc/kubernetes/kubeconfig
-    - --leader-elect=true
-    volumeMounts:
-    - name: kubernetes
-      mountPath: /etc/kubernetes
-      readOnly: true
-  hostNetwork: true
-  volumes:
-  - name: kubernetes
-    hostPath:
-      path: /etc/kubernetes

resources/calico/calico-bgp-peers.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico BGP Peers
-kind: CustomResourceDefinition
-metadata:
-  name: bgppeers.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: BGPPeer
-    plural: bgppeers
-    singular: bgppeer

resources/calico/calico-cluster-role.yaml

@@ -1,53 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: calico-node
-  namespace: kube-system
-rules:
-  - apiGroups: [""]
-    resources:
-      - namespaces
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: [""]
-    resources:
-      - pods/status
-    verbs:
-      - update
-  - apiGroups: [""]
-    resources:
-      - pods
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: [""]
-    resources:
-      - nodes
-    verbs:
-      - get
-      - list
-      - update
-      - watch
-  - apiGroups: ["extensions"]
-    resources:
-      - networkpolicies
-    verbs:
-      - get
-      - list
-      - watch
-  - apiGroups: ["crd.projectcalico.org"]
-    resources:
-      - globalfelixconfigs
-      - bgppeers
-      - globalbgpconfigs
-      - ippools
-      - globalnetworkpolicies
-    verbs:
-      - create
-      - get
-      - list
-      - update
-      - watch

resources/calico/calico-config.yaml

@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: calico-config
-  namespace: kube-system
-data:
-  # The CNI network configuration to install on each node.
-  cni_network_config: |-
-    {
-        "name": "k8s-pod-network",
-        "cniVersion": "0.3.0",
-        "type": "calico",
-        "log_level": "info",
-        "datastore_type": "kubernetes",
-        "nodename": "__KUBERNETES_NODE_NAME__",
-        "mtu": ${network_mtu},
-        "ipam": {
-            "type": "host-local",
-            "subnet": "usePodCidr"
-        },
-        "policy": {
-            "type": "k8s",
-            "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
-        },
-        "kubernetes": {
-            "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
-            "kubeconfig": "__KUBECONFIG_FILEPATH__"
-        }
-    }

resources/calico/calico-gloabl-felix-configs.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global Felix Configuration
-kind: CustomResourceDefinition
-metadata:
-   name: globalfelixconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalFelixConfig
-    plural: globalfelixconfigs
-    singular: globalfelixconfig

resources/calico/calico-global-bgp-configs.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global BGP Configuration
-kind: CustomResourceDefinition
-metadata:
-  name: globalbgpconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalBGPConfig
-    plural: globalbgpconfigs
-    singular: globalbgpconfig

resources/calico/calico-ip-pools.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico IP Pools
-kind: CustomResourceDefinition
-metadata:
-  name: ippools.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: IPPool
-    plural: ippools
-    singular: ippool

resources/calico/calico-network-policies.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global Network Policies
-kind: CustomResourceDefinition
-metadata:
-  name: globalnetworkpolicies.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalNetworkPolicy
-    plural: globalnetworkpolicies
-    singular: globalnetworkpolicy

resources/calico/calico.yaml

@@ -1,134 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: calico-node
-  namespace: kube-system
-  labels:
-    k8s-app: calico-node
-spec:
-  selector:
-    matchLabels:
-      k8s-app: calico-node
-  template:
-    metadata:
-      labels:
-        k8s-app: calico-node
-    spec:
-      hostNetwork: true
-      serviceAccountName: calico-node
-      tolerations:
-        # Allow the pod to run on master nodes
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
-      containers:
-        - name: calico-node
-          image: ${calico_image}
-          env:
-            # Use Kubernetes API as the backing datastore.
-            - name: DATASTORE_TYPE
-              value: "kubernetes"
-            # Enable felix info logging.
-            - name: FELIX_LOGSEVERITYSCREEN
-              value: "info"
-            # Cluster type to identify the deployment type
-            - name: CLUSTER_TYPE
-              value: "k8s,bgp"
-            # Disable file logging so `kubectl logs` works.
-            - name: CALICO_DISABLE_FILE_LOGGING
-              value: "true"
-            # Set Felix endpoint to host default action to ACCEPT.
-            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
-              value: "ACCEPT"
-            # Disable IPV6 on Kubernetes.
-            - name: FELIX_IPV6SUPPORT
-              value: "false"
-            # Set MTU for tunnel device used if ipip is enabled
-            - name: FELIX_IPINIPMTU
-              value: "${network_mtu}"
-            # Wait for the datastore.
-            - name: WAIT_FOR_DATASTORE
-              value: "true"
-            # The Calico IPv4 pool CIDR (should match `--cluster-cidr`).
-            - name: CALICO_IPV4POOL_CIDR
-              value: "${pod_cidr}"
-            # Enable IPIP
-            - name: CALICO_IPV4POOL_IPIP
-              value: "Always"
-            # Enable IP-in-IP within Felix.
-            - name: FELIX_IPINIPENABLED
-              value: "true"
-            # Set node name based on k8s nodeName.
-            - name: NODENAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: FELIX_HEALTHENABLED
-              value: "true"
-          securityContext:
-            privileged: true
-          resources:
-            requests:
-              cpu: 250m
-          livenessProbe:
-            httpGet:
-              path: /liveness
-              port: 9099
-            periodSeconds: 10
-            initialDelaySeconds: 10
-            failureThreshold: 6
-          readinessProbe:
-            httpGet:
-              path: /readiness
-              port: 9099
-            periodSeconds: 10
-          volumeMounts:
-            - mountPath: /lib/modules
-              name: lib-modules
-              readOnly: true
-            - mountPath: /var/run/calico
-              name: var-run-calico
-              readOnly: false
-        # Install Calico CNI binaries and CNI network config file on nodes
-        - name: install-cni
-          image: ${calico_cni_image}
-          command: ["/install-cni.sh"]
-          env:
-            - name: CNI_NETWORK_CONFIG
-              valueFrom:
-                configMapKeyRef:
-                  name: calico-config
-                  key: cni_network_config
-            - name: CNI_NET_DIR
-              value: "/etc/kubernetes/cni/net.d"
-            # Set node name based on k8s nodeName
-            - name: KUBERNETES_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-      terminationGracePeriodSeconds: 0
-      volumes:
-        - name: lib-modules
-          hostPath:
-            path: /lib/modules
-        - name: var-run-calico
-          hostPath:
-            path: /var/run/calico
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-net-dir
-          hostPath:
-            path: /etc/kubernetes/cni/net.d
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/cilium/cluster-role-binding.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-operator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-operator
+subjects:
+- kind: ServiceAccount
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cilium-agent
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cilium-agent
+subjects:
+- kind: ServiceAccount
+  name: cilium-agent
+  namespace: kube-system
+

resources/cilium/cluster-role.yaml

@@ -0,0 +1,188 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-operator
+rules:
+- apiGroups:
+  - ""
+  resources:
+  # to automatically delete [core|kube]dns pods so that are starting to being
+  # managed by Cilium
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  # to perform LB IP allocation for BGP
+  - services/status
+  verbs:
+  - update
+- apiGroups:
+  - ""
+  resources:
+  # to perform the translation of a CNP that contains `ToGroup` to its endpoints
+  - services
+  - endpoints
+  # to check apiserver connectivity
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumnetworkpolicies/finalizers
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies/finalizers
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumendpoints/finalizers
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumnodes/finalizers
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumidentities/finalizers
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumlocalredirectpolicies/finalizers
+  - ciliumendpointslices
+  - ciliumloadbalancerippools
+  - ciliumloadbalancerippools/status
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - update
+  - watch
+# Cilium leader elects if among multiple operator replicas
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cilium-agent
+rules:
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - discovery.k8s.io
+  resources:
+  - endpointslices
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - services
+  - pods
+  - endpoints
+  - nodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - cilium.io
+  resources:
+  - ciliumnetworkpolicies
+  - ciliumnetworkpolicies/status
+  - ciliumclusterwidenetworkpolicies
+  - ciliumclusterwidenetworkpolicies/status
+  - ciliumendpoints
+  - ciliumendpoints/status
+  - ciliumnodes
+  - ciliumnodes/status
+  - ciliumidentities
+  - ciliumidentities/status
+  - ciliumlocalredirectpolicies
+  - ciliumlocalredirectpolicies/status
+  - ciliumegressnatpolicies
+  - ciliumendpointslices
+  - ciliumcidrgroups
+  - ciliuml2announcementpolicies
+  - ciliuml2announcementpolicies/status
+  - ciliumpodippools
+  verbs:
+  - '*'
+

resources/cilium/config.yaml

@@ -0,0 +1,175 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cilium
+  namespace: kube-system
+data:
+  # Identity allocation mode selects how identities are shared between cilium
+  # nodes by setting how they are stored. The options are "crd" or "kvstore".
+  # - "crd" stores identities in kubernetes as CRDs (custom resource definition).
+  #   These can be queried with:
+  #     kubectl get ciliumid
+  # - "kvstore" stores identities in a kvstore, etcd or consul, that is
+  #   configured below. Cilium versions before 1.6 supported only the kvstore
+  #   backend. Upgrades from these older cilium versions should continue using
+  #   the kvstore by commenting out the identity-allocation-mode below, or
+  #   setting it to "kvstore".
+  identity-allocation-mode: crd
+  cilium-endpoint-gc-interval: "5m0s"
+  nodes-gc-interval: "5m0s"
+
+  # If you want to run cilium in debug mode change this value to true
+  debug: "false"
+  # The agent can be put into the following three policy enforcement modes
+  # default, always and never.
+  # https://docs.cilium.io/en/latest/policy/intro/#policy-enforcement-modes
+  enable-policy: "default"
+
+  # Prometheus
+  # enable-metrics: "true"
+  # prometheus-serve-addr: ":foo"
+  # operator-prometheus-serve-addr: ":bar"
+
+  # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
+  # address.
+  enable-ipv4: "true"
+
+  # Enable IPv6 addressing. If enabled, all endpoints are allocated an IPv6
+  # address.
+  enable-ipv6: "false"
+
+  # Enable probing for a more efficient clock source for the BPF datapath
+  enable-bpf-clock-probe: "true"
+
+  # Enable use of transparent proxying mechanisms (Linux 5.7+)
+  enable-bpf-tproxy: "false"
+
+  # If you want cilium monitor to aggregate tracing for packets, set this level
+  # to "low", "medium", or "maximum". The higher the level, the less packets
+  # that will be seen in monitor output.
+  monitor-aggregation: medium
+
+  # The monitor aggregation interval governs the typical time between monitor
+  # notification events for each allowed connection.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-interval: 5s
+
+  # The monitor aggregation flags determine which TCP flags which, upon the
+  # first observation, cause monitor notifications to be generated.
+  #
+  # Only effective when monitor aggregation is set to "medium" or higher.
+  monitor-aggregation-flags: all
+
+  # Specifies the ratio (0.0-1.0) of total system memory to use for dynamic
+  # sizing of the TCP CT, non-TCP CT, NAT and policy BPF maps.
+  bpf-map-dynamic-size-ratio: "0.0025"
+  # bpf-policy-map-max specified the maximum number of entries in endpoint
+  # policy map (per endpoint)
+  bpf-policy-map-max: "16384"
+  # bpf-lb-map-max specifies the maximum number of entries in bpf lb service,
+  # backend and affinity maps.
+  bpf-lb-map-max: "65536"
+
+  # Pre-allocation of map entries allows per-packet latency to be reduced, at
+  # the expense of up-front memory allocation for the entries in the maps. The
+  # default value below will minimize memory usage in the default installation;
+  # users who are sensitive to latency may consider setting this to "true".
+  #
+  # This option was introduced in Cilium 1.4. Cilium 1.3 and earlier ignore
+  # this option and behave as though it is set to "true".
+  #
+  # If this value is modified, then during the next Cilium startup the restore
+  # of existing endpoints and tracking of ongoing connections may be disrupted.
+  # As a result, reply packets may be dropped and the load-balancing decisions
+  # for established connections may change.
+  #
+  # If this option is set to "false" during an upgrade from 1.3 or earlier to
+  # 1.4 or later, then it may cause one-time disruptions during the upgrade.
+  preallocate-bpf-maps: "false"
+
+  # Name of the cluster. Only relevant when building a mesh of clusters.
+  cluster-name: default
+  # Unique ID of the cluster. Must be unique across all conneted clusters and
+  # in the range of 1 and 255. Only relevant when building a mesh of clusters.
+  cluster-id: "0"
+
+  # Encapsulation mode for communication between nodes
+  # Possible values:
+  #   - disabled
+  #   - vxlan (default)
+  #   - geneve
+  routing-mode: "tunnel"
+  tunnel: vxlan
+  # Enables L7 proxy for L7 policy enforcement and visibility
+  enable-l7-proxy: "true"
+
+  auto-direct-node-routes: "false"
+
+  # enableXTSocketFallback enables the fallback compatibility solution
+  # when the xt_socket kernel module is missing and it is needed for
+  # the datapath L7 redirection to work properly.  See documentation
+  # for details on when this can be disabled:
+  # http://docs.cilium.io/en/latest/install/system_requirements/#admin-kernel-version.
+  enable-xt-socket-fallback: "true"
+
+  # installIptablesRules enables installation of iptables rules to allow for
+  # TPROXY (L7 proxy injection), itpables based masquerading and compatibility
+  # with kube-proxy. See documentation for details on when this can be
+  # disabled.
+  install-iptables-rules: "true"
+
+  # masquerade traffic leaving the node destined for outside
+  enable-ipv4-masquerade: "true"
+  enable-ipv6-masquerade: "false"
+
+  # bpfMasquerade enables masquerading with BPF instead of iptables
+  enable-bpf-masquerade: "true"
+
+  # kube-proxy
+  kube-proxy-replacement:  "true"
+  kube-proxy-replacement-healthz-bind-address: ":10256"
+  enable-session-affinity: "true"
+
+  # ClusterIPs from host namespace
+  bpf-lb-sock: "true"
+  # ClusterIPs from external nodes
+  bpf-lb-external-clusterip: "true"
+
+  # NodePort
+  enable-node-port: "true"
+  enable-health-check-nodeport: "false"
+
+  # ExternalIPs
+  enable-external-ips: "true"
+
+  # HostPort
+  enable-host-port: "true"
+
+  # IPAM
+  ipam: "cluster-pool"
+  disable-cnp-status-updates: "true"
+  cluster-pool-ipv4-cidr: "${pod_cidr}"
+  cluster-pool-ipv4-mask-size: "24"
+
+  # Health
+  agent-health-port: "9876"
+  enable-health-checking: "true"
+  enable-endpoint-health-checking: "true"
+
+  # Identity
+  enable-well-known-identities: "false"
+  enable-remote-node-identity: "true"
+
+  # Misc
+  enable-bandwidth-manager: "false"
+  enable-local-redirect-policy: "false"
+  policy-audit-mode: "false"
+  operator-api-serve-addr: "127.0.0.1:9234"
+  enable-l2-neigh-discovery: "true"
+  enable-k8s-terminating-endpoint: "true"
+  enable-k8s-networkpolicy: "true"
+  external-envoy-proxy: "false"
+  write-cni-conf-when-ready: /host/etc/cni/net.d/05-cilium.conflist
+  cni-exclusive: "true"
+  cni-log-file: "/var/run/cilium/cilium-cni.log"

resources/cilium/daemonset.yaml

@@ -0,0 +1,219 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cilium
+  namespace: kube-system
+  labels:
+    k8s-app: cilium
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cilium-agent
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: cilium-agent
+    spec:
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      serviceAccountName: cilium-agent
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+      # Cilium v1.13.1 starts installing CNI plugins in yet another init container
+      # https://github.com/cilium/cilium/pull/24075
+      - name: install-cni
+        image: ${cilium_agent_image}
+        command:
+          - /install-plugin.sh
+        securityContext:
+          privileged: true
+          capabilities:
+            drop:
+              - ALL
+        volumeMounts:
+          - name: cni-bin-dir
+            mountPath: /host/opt/cni/bin
+
+      # Required to mount cgroup2 filesystem on the underlying Kubernetes node.
+      # We use nsenter command with host's cgroup and mount namespaces enabled.
+      - name: mount-cgroup
+        image: ${cilium_agent_image}
+        command:
+          - sh
+          - -ec
+          # The statically linked Go program binary is invoked to avoid any
+          # dependency on utilities like sh and mount that can be missing on certain
+          # distros installed on the underlying host. Copy the binary to the
+          # same directory where we install cilium cni plugin so that exec permissions
+          # are available.
+          - 'cp /usr/bin/cilium-mount /hostbin/cilium-mount && nsenter --cgroup=/hostproc/1/ns/cgroup --mount=/hostproc/1/ns/mnt "$${BIN_PATH}/cilium-mount" $CGROUP_ROOT; rm /hostbin/cilium-mount'
+        env:
+          - name: CGROUP_ROOT
+            value: /run/cilium/cgroupv2
+          - name: BIN_PATH
+            value: /opt/cni/bin
+        securityContext:
+          privileged: true
+        volumeMounts:
+          - name: hostproc
+            mountPath: /hostproc
+          - name: cni-bin-dir
+            mountPath: /hostbin
+
+      - name: clean-cilium-state
+        image: ${cilium_agent_image}
+        command:
+        - /init-container.sh
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        # Required to mount cgroup filesystem from the host to cilium agent pod
+        - name: cilium-cgroup
+          mountPath: /run/cilium/cgroupv2
+          mountPropagation: HostToContainer
+
+      containers:
+      - name: cilium-agent
+        image: ${cilium_agent_image}
+        command:
+        - cilium-agent
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        ports:
+          # Not yet used, prefer exec's
+          - name: health
+            protocol: TCP
+            containerPort: 9876
+        lifecycle:
+          preStop:
+            exec:
+              command:
+              - /cni-uninstall.sh
+        securityContext:
+          privileged: true
+        livenessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 30
+          initialDelaySeconds: 120
+          successThreshold: 1
+          failureThreshold: 10
+          timeoutSeconds: 5
+        readinessProbe:
+          exec:
+            command:
+            - cilium
+            - status
+            - --brief
+          periodSeconds: 20
+          initialDelaySeconds: 5
+          successThreshold: 1
+          failureThreshold: 3
+          timeoutSeconds: 5
+        volumeMounts:
+        # Load kernel modules
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+        # Keep state between restarts
+        - name: var-run-cilium
+          mountPath: /var/run/cilium
+        - name: sys-fs-bpf
+          mountPath: /sys/fs/bpf
+          mountPropagation: Bidirectional
+        # Configuration
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+        # Install config on host
+        - name: cni-conf-dir
+          mountPath: /host/etc/cni/net.d
+      terminationGracePeriodSeconds: 1
+      volumes:
+      # Load kernel modules
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      # Access iptables concurrently with other processes (e.g. kube-proxy)
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock
+      # Keep state between restarts
+      - name: var-run-cilium
+        hostPath:
+          path: /var/run/cilium
+          type: DirectoryOrCreate
+      # Keep state between restarts for bpf maps
+      - name: sys-fs-bpf
+        hostPath:
+          path: /sys/fs/bpf
+          type: DirectoryOrCreate
+      # Mount host cgroup2 filesystem
+      - name: hostproc
+        hostPath:
+          path: /proc
+          type: Directory
+      - name: cilium-cgroup
+        hostPath:
+          path: /run/cilium/cgroupv2
+          type: DirectoryOrCreate
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium
+      # Install CNI plugin and config on host
+      - name: cni-bin-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path:  /opt/cni/bin
+      - name: cni-conf-dir
+        hostPath:
+          type: DirectoryOrCreate
+          path: /etc/cni/net.d
+

resources/cilium/deployment.yaml

@@ -0,0 +1,103 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      name: cilium-operator
+  template:
+    metadata:
+      labels:
+        name: cilium-operator
+    spec:
+      hostNetwork: true
+      priorityClassName: system-cluster-critical
+      serviceAccountName: cilium-operator
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      containers:
+      - name: cilium-operator
+        image: ${cilium_operator_image}
+        command:
+        - cilium-operator-generic
+        args:
+        - --config-dir=/tmp/cilium/config-map
+        - --debug=$(CILIUM_DEBUG)
+        env:
+        - name: K8S_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: CILIUM_K8S_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: KUBERNETES_SERVICE_HOST
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-host
+        - name: KUBERNETES_SERVICE_PORT
+          valueFrom:
+            configMapKeyRef:
+              name: in-cluster
+              key: apiserver-port
+        - name: CILIUM_DEBUG
+          valueFrom:
+            configMapKeyRef:
+              name: cilium
+              key: debug
+              optional: true
+        ports:
+          - name: health
+            protocol: TCP
+            containerPort: 9234
+        livenessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          timeoutSeconds: 3
+        readinessProbe:
+          httpGet:
+            scheme: HTTP
+            host: 127.0.0.1
+            port: 9234
+            path: /healthz
+          periodSeconds: 15
+          timeoutSeconds: 3
+          failureThreshold: 5
+        volumeMounts:
+        - name: config
+          mountPath: /tmp/cilium/config-map
+          readOnly: true
+      topologySpreadConstraints:
+        - topologyKey: kubernetes.io/hostname
+          labelSelector:
+            matchLabels:
+              name: cilium-operator
+          maxSkew: 1
+          whenUnsatisfiable: DoNotSchedule
+      volumes:
+      # Read configuration
+      - name: config
+        configMap:
+          name: cilium

resources/cilium/service-account.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-operator
+  namespace: kube-system
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cilium-agent
+  namespace: kube-system
+

resources/coredns/cluster-role-binding.yaml

@@ -0,0 +1,16 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system:coredns
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:coredns
+subjects:
+  - kind: ServiceAccount
+    name: coredns
+    namespace: kube-system

resources/coredns/cluster-role.yaml

@@ -0,0 +1,27 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:coredns
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+rules:
+  - apiGroups: [""]
+    resources:
+      - endpoints
+      - services
+      - pods
+      - namespaces
+    verbs:
+      - list
+      - watch
+  - apiGroups: ["discovery.k8s.io"]
+    resources:
+      - endpointslices
+    verbs:
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get

resources/coredns/config.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+          lameduck 5s
+        }
+        ready
+        log . {
+            class error
+        }
+        kubernetes ${cluster_domain_suffix} in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf
+        cache 30
+        loop
+        reload
+        loadbalance
+    }

resources/coredns/deployment.yaml

@@ -0,0 +1,109 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coredns
+  namespace: kube-system
+  labels:
+    k8s-app: coredns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  replicas: ${control_plane_replicas}
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      tier: control-plane
+      k8s-app: coredns
+  template:
+    metadata:
+      labels:
+        tier: control-plane
+        k8s-app: coredns
+    spec:
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            preference:
+              matchExpressions:
+              - key: node.kubernetes.io/controller
+                operator: Exists
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: tier
+                  operator: In
+                  values:
+                  - control-plane
+                - key: k8s-app
+                  operator: In
+                  values:
+                  - coredns
+              topologyKey: kubernetes.io/hostname
+      priorityClassName: system-cluster-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: coredns
+      tolerations:
+        - key: node-role.kubernetes.io/controller
+          effect: NoSchedule
+      containers:
+        - name: coredns
+          image: ${coredns_image}
+          resources:
+            limits:
+              memory: 170Mi
+            requests:
+              cpu: 100m
+              memory: 70Mi
+          args: [ "-conf", "/etc/coredns/Corefile" ]
+          volumeMounts:
+            - name: config
+              mountPath: /etc/coredns
+              readOnly: true
+          ports:
+            - name: dns
+              protocol: UDP
+              containerPort: 53
+            - name: dns-tcp
+              protocol: TCP
+              containerPort: 53
+            - name: metrics
+              protocol: TCP
+              containerPort: 9153
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 5
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8181
+              scheme: HTTP
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - NET_BIND_SERVICE
+              drop:
+              - all
+            readOnlyRootFilesystem: true
+      dnsPolicy: Default
+      volumes:
+        - name: config
+          configMap:
+            name: coredns
+            items:
+            - key: Corefile
+              path: Corefile

resources/coredns/service-account.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: calico-node
+  name: coredns
   namespace: kube-system

resources/coredns/service.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: coredns
+  namespace: kube-system
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9153"
+  labels:
+    k8s-app: coredns
+    kubernetes.io/name: "CoreDNS"
+spec:
+  selector:
+    k8s-app: coredns
+  clusterIP: ${cluster_dns_service_ip}
+  ports:
+    - name: dns
+      port: 53
+      protocol: UDP
+    - name: dns-tcp
+      port: 53
+      protocol: TCP

resources/etcd/bootstrap-etcd-service.json

@@ -1,26 +0,0 @@
-{
-  "apiVersion": "v1",
-  "kind": "Service",
-  "metadata": {
-    "name": "bootstrap-etcd-service",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "selector": {
-      "k8s-app": "boot-etcd"
-    },
-    "clusterIP": "${bootstrap_etcd_service_ip}",
-    "ports": [
-      {
-        "name": "client",
-        "port": 12379,
-        "protocol": "TCP"
-      },
-      {
-        "name": "peers",
-        "port": 12380,
-        "protocol": "TCP"
-      }
-    ]
-  }
-}

resources/etcd/migrate-etcd-cluster.json

@@ -1,36 +0,0 @@
-{
-  "apiVersion": "etcd.database.coreos.com/v1beta2",
-  "kind": "EtcdCluster",
-  "metadata": {
-    "name": "kube-etcd",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "size": 1,
-    "version": "v3.1.8",
-    "pod": {
-      "nodeSelector": {
-        "node-role.kubernetes.io/master": ""
-      },
-      "tolerations": [
-        {
-          "key": "node-role.kubernetes.io/master",
-          "operator": "Exists",
-          "effect": "NoSchedule"
-        }
-      ]
-    },
-    "selfHosted": {
-      "bootMemberClientEndpoint": "https://${bootstrap_etcd_service_ip}:12379"
-    },
-    "TLS": {
-      "static": {
-        "member": {
-          "peerSecret": "etcd-peer-tls",
-          "serverSecret": "etcd-server-tls"
-        },
-        "operatorSecret": "etcd-client-tls"
-      }
-    }
-  }
-}

resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-etcd
-  namespace: kube-system
-  labels:
-    k8s-app: boot-etcd
-spec:
-  containers:
-  - name: etcd
-    image: ${etcd_image}
-    command:
-    - /usr/local/bin/etcd
-    - --name=boot-etcd
-    - --listen-client-urls=https://0.0.0.0:12379
-    - --listen-peer-urls=https://0.0.0.0:12380
-    - --advertise-client-urls=https://${bootstrap_etcd_service_ip}:12379
-    - --initial-advertise-peer-urls=https://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster=boot-etcd=https://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster-token=bootkube
-    - --initial-cluster-state=new
-    - --data-dir=/var/etcd/data
-    - --peer-client-cert-auth=true
-    - --peer-trusted-ca-file=/etc/kubernetes/secrets/etcd/peer-ca.crt
-    - --peer-cert-file=/etc/kubernetes/secrets/etcd/peer.crt
-    - --peer-key-file=/etc/kubernetes/secrets/etcd/peer.key
-    - --client-cert-auth=true
-    - --trusted-ca-file=/etc/kubernetes/secrets/etcd/server-ca.crt
-    - --cert-file=/etc/kubernetes/secrets/etcd/server.crt
-    - --key-file=/etc/kubernetes/secrets/etcd/server.key
-    volumeMounts:
-      - mountPath: /etc/kubernetes/secrets
-        name: secrets
-        readOnly: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets 
-  hostNetwork: true
-  restartPolicy: Never
-  dnsPolicy: ClusterFirstWithHostNet 

resources/experimental/manifests/etcd-client-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-client-tls
-  namespace: kube-system
-type: Opaque
-data:
-  etcd-client-ca.crt: ${etcd_ca_cert}
-  etcd-client.crt: ${etcd_client_cert}
-  etcd-client.key: ${etcd_client_key}

resources/experimental/manifests/etcd-operator.yaml

@@ -1,46 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: etcd-operator
-  namespace: kube-system
-  labels:
-    k8s-app: etcd-operator
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: etcd-operator
-  template:
-    metadata:
-      labels:
-        k8s-app: etcd-operator
-    spec:
-      containers:
-      - name: etcd-operator
-        image: ${etcd_operator_image}
-        command:
-        - /usr/local/bin/etcd-operator
-        - --analytics=false
-        env:
-        - name: MY_POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: MY_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-      maxSurge: 1

resources/experimental/manifests/etcd-peer-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-peer-tls
-  namespace: kube-system
-type: Opaque
-data:
-  peer-ca.crt: ${etcd_ca_cert}
-  peer.crt: ${etcd_peer_cert}
-  peer.key: ${etcd_peer_key}

resources/experimental/manifests/etcd-server-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-server-tls
-  namespace: kube-system
-type: Opaque
-data:
-  server-ca.crt: ${etcd_ca_cert}
-  server.crt: ${etcd_server_cert}
-  server.key: ${etcd_server_key}

resources/experimental/manifests/etcd-service.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-service
-  namespace: kube-system
-  # This alpha annotation will retain the endpoints even if the etcd pod isn't ready.
-  # This feature is always enabled in endpoint controller in k8s even it is alpha.
-  annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
-spec:
-  selector:
-    app: etcd
-    etcd_cluster: kube-etcd
-  clusterIP: ${etcd_service_ip}
-  ports:
-  - name: client
-    port: 2379
-    protocol: TCP

resources/experimental/manifests/kube-etcd-network-checkpointer.yaml

@@ -1,62 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: kube-etcd-network-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-etcd-network-checkpointer
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-etcd-network-checkpointer
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-etcd-network-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-    spec:
-      containers:
-      - image: ${etcd_checkpointer_image}
-        name: kube-etcd-network-checkpointer
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /etc/kubernetes/selfhosted-etcd
-          name: checkpoint-dir
-          readOnly: false
-        - mountPath: /var/etcd
-          name: etcd-dir
-          readOnly: false
-        - mountPath: /var/lock
-          name: var-lock
-          readOnly: false
-        command:
-        - /usr/bin/flock
-        - /var/lock/kenc.lock
-        - -c
-        - "kenc -r -m iptables && kenc -m iptables"
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: checkpoint-dir
-        hostPath:
-          path: /etc/kubernetes/checkpoint-iptables
-      - name: etcd-dir
-        hostPath:
-          path: /var/etcd
-      - name: var-lock
-        hostPath:
-          path: /var/lock
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/flannel/cluster-role-binding.yaml

@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: calico-node
+  name: flannel
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: calico-node
+  name: flannel
 subjects:
 - kind: ServiceAccount
-  name: calico-node
+  name: flannel
   namespace: kube-system

resources/flannel/cluster-role.yaml

@@ -0,0 +1,24 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch

resources/flannel/config.yaml

@@ -1,7 +1,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: kube-flannel-cfg
+  name: flannel-config
   namespace: kube-system
   labels:
     tier: node
@@ -31,6 +31,7 @@ data:
     {
       "Network": "${pod_cidr}",
       "Backend": {
-        "Type": "vxlan"
+        "Type": "vxlan",
+        "Port": 8472
       }
     }

resources/flannel/daemonset.yaml

@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: flannel
+  namespace: kube-system
+  labels:
+    k8s-app: flannel
+spec:
+  selector:
+    matchLabels:
+      k8s-app: flannel
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: flannel
+    spec:
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      serviceAccountName: flannel
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      initContainers:
+        - name: install-cni
+          image: ${flannel_cni_image}
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: flannel-config
+                  key: cni-conf.json
+          volumeMounts:
+            - name: cni-bin-dir
+              mountPath: /host/opt/cni/bin/
+            - name: cni-conf-dir
+              mountPath: /host/etc/cni/net.d
+      containers:
+        - name: flannel
+          image: ${flannel_image}
+          command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr", "--iface=$(POD_IP)"]
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 100m
+          volumeMounts:
+            - name: flannel-config
+              mountPath: /etc/kube-flannel/
+            - name: run-flannel
+              mountPath: /run/flannel
+            - name: xtables-lock
+              mountPath: /run/xtables.lock
+      volumes:
+        - name: flannel-config
+          configMap:
+            name: flannel-config
+        - name: run-flannel
+          hostPath:
+            path: /run/flannel
+        # Used by install-cni
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-conf-dir
+          hostPath:
+            type: DirectoryOrCreate
+            path: /etc/cni/net.d
+        # Access iptables concurrently
+        - name: xtables-lock
+          hostPath:
+            type: FileOrCreate
+            path: /run/xtables.lock

resources/flannel/kube-flannel.yaml

@@ -1,81 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: kube-flannel
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: flannel
-spec:
-  selector:
-    matchLabels:
-      tier: node
-      k8s-app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        k8s-app: flannel
-    spec:
-      containers:
-      - name: kube-flannel
-        image: ${flannel_image}
-        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr", "--iface=$(POD_IP)"]
-        securityContext:
-          privileged: true
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        volumeMounts:
-        - name: run
-          mountPath: /run
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      - name: install-cni
-        image: ${flannel_cni_image}
-        command: ["/install-cni.sh"]
-        env:
-        - name: CNI_NETWORK_CONFIG
-          valueFrom:
-            configMapKeyRef:
-              name: kube-flannel-cfg
-              key: cni-conf.json
-        volumeMounts:
-        - name: cni
-          mountPath: /host/etc/cni/net.d
-        - name: host-cni-bin
-          mountPath: /host/opt/cni/bin/
-      hostNetwork: true
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-        - name: run
-          hostPath:
-            path: /run
-        - name: cni
-          hostPath:
-            path: /etc/kubernetes/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
-        - name: host-cni-bin
-          hostPath:
-            path: /opt/cni/bin
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/flannel/service-account.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  name: flannel
   namespace: kube-system
-  name: pod-checkpointer

resources/kube-proxy/kube-proxy.yaml

@@ -0,0 +1,99 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kube-proxy
+  namespace: kube-system
+  labels:
+    tier: node
+    k8s-app: kube-proxy
+spec:
+  selector:
+    matchLabels:
+      tier: node
+      k8s-app: kube-proxy
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        tier: node
+        k8s-app: kube-proxy
+    spec:
+      hostNetwork: true
+      priorityClassName: system-node-critical
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+      serviceAccountName: kube-proxy
+      tolerations:
+      - key: node-role.kubernetes.io/controller
+        operator: Exists
+      - key: node.kubernetes.io/not-ready
+        operator: Exists
+      %{~ for key in daemonset_tolerations ~}
+      - key: ${key}
+        operator: Exists
+      %{~ endfor ~}
+      containers:
+      - name: kube-proxy
+        image: ${kube_proxy_image}
+        command:
+        - kube-proxy
+        - --cluster-cidr=${pod_cidr}
+        - --hostname-override=$(NODE_NAME)
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --metrics-bind-address=0.0.0.0
+        - --proxy-mode=ipvs
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        ports:
+        - name: metrics
+          containerPort: 10249
+        - name: health
+          containerPort: 10256
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10256
+          initialDelaySeconds: 15
+          timeoutSeconds: 15
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: kubeconfig
+          mountPath: /etc/kubernetes
+          readOnly: true
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: etc-ssl
+          mountPath: /etc/ssl/certs
+          readOnly: true
+        - name: etc-pki
+          mountPath: /etc/pki
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+      volumes:
+      - name: kubeconfig
+        configMap:
+          name: kubeconfig-in-cluster
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: etc-ssl
+        hostPath:
+          path: /etc/ssl/certs
+      - name: etc-pki
+        hostPath:
+          path: /etc/pki
+      # Access iptables concurrently
+      - name: xtables-lock
+        hostPath:
+          type: FileOrCreate
+          path: /run/xtables.lock

resources/kubeconfig-admin

@@ -1,17 +1,18 @@
 apiVersion: v1
 kind: Config
 clusters:
-- name: ${name}-cluster
+- name: ${name}
   cluster:
     server: ${server}
     certificate-authority-data: ${ca_cert}
 users:
-- name: ${name}-user
+- name: ${name}
   user:
     client-certificate-data: ${kubelet_cert}
     client-key-data: ${kubelet_key}
+current-context: ${name}
 contexts:
-- name: ${name}-context
+- name: ${name}
   context:
-    cluster: ${name}-cluster
-    user: ${name}-user
+    cluster: ${name}
+    user: ${name}

resources/kubeconfig-bootstrap

@@ -8,8 +8,7 @@ clusters:
 users:
 - name: kubelet
   user:
-    client-certificate-data: ${kubelet_cert}
-    client-key-data: ${kubelet_key}
+    token: ${token_id}.${token_secret}
 contexts:
 - context:
     cluster: local

resources/manifests/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Bind system:bootstrappers to ClusterRole for node bootstrap
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-bootstrapper
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve new CSRs from "system:bootstrappers" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-new
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve renewal CSRs from "system:nodes" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-renew
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/bootstrap-token.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>"
+  name: bootstrap-token-${token_id}
+  namespace: kube-system
+stringData:
+  description: "Typhoon generated bootstrap token"
+  token-id: ${token_id}
+  token-secret: ${token_secret}
+  usage-bootstrap-authentication: "true"

resources/manifests/in-cluster.yaml

@@ -0,0 +1,10 @@
+# in-cluster ConfigMap is for control plane components that must reach
+# kube-apiserver before service IPs are available (e.g. 10.3.0.1)
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: in-cluster
+  namespace: kube-system
+data:
+  apiserver-host: ${apiserver_host}
+  apiserver-port: "${apiserver_port}"

resources/manifests/kube-apiserver-secret.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-type: Opaque
-data:
-  apiserver.key: ${apiserver_key}
-  apiserver.crt: ${apiserver_cert}
-  service-account.pub: ${serviceaccount_pub}
-  ca.crt: ${ca_cert}
-  etcd-client-ca.crt: ${etcd_ca_cert}
-  etcd-client.crt: ${etcd_client_cert}
-  etcd-client.key: ${etcd_client_key}

resources/manifests/kube-apiserver.yaml

@@ -1,86 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-apiserver
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-apiserver
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-apiserver
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-    spec:
-      containers:
-      - name: kube-apiserver
-        image: ${hyperkube_image}
-        command:
-        - /hyperkube
-        - apiserver
-        - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
-        - --advertise-address=$(POD_IP)
-        - --allow-privileged=true
-        - --anonymous-auth=false
-        - --authorization-mode=RBAC
-        - --bind-address=0.0.0.0
-        - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --cloud-provider=${cloud_provider}
-        - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-        - --etcd-quorum-read=true
-        - --etcd-servers=${etcd_servers}
-        - --insecure-port=0
-        - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-        - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-        - --secure-port=443
-        - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
-        - --service-cluster-ip-range=${service_cidr}
-        - --storage-backend=etcd3
-        - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-        - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
-        env:
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        volumeMounts:
-        - mountPath: /etc/ssl/certs
-          name: ssl-certs-host
-          readOnly: true
-        - mountPath: /etc/kubernetes/secrets
-          name: secrets
-          readOnly: true
-        - mountPath: /var/lock
-          name: var-lock
-          readOnly: false
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: ssl-certs-host
-        hostPath:
-          path: /usr/share/ca-certificates
-      - name: secrets
-        secret:
-          secretName: kube-apiserver
-      - name: var-lock
-        hostPath:
-          path: /var/lock
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/manifests/kube-controller-manager-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager

resources/manifests/kube-controller-manager-secret.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-type: Opaque
-data:
-  service-account.key: ${serviceaccount_key}
-  ca.crt: ${ca_cert}

resources/manifests/kube-controller-manager.yaml

@@ -1,79 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-controller-manager
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-controller-manager
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-controller-manager
-              topologyKey: kubernetes.io/hostname
-      containers:
-      - name: kube-controller-manager
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - controller-manager
-        - --allocate-node-cidrs=true
-        - --cloud-provider=${cloud_provider}
-        - --cluster-cidr=${pod_cidr}
-        - --configure-cloud-routes=false
-        - --leader-elect=true
-        - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 10252  # Note: Using default port. Update if --port option is set differently.
-          initialDelaySeconds: 15
-          timeoutSeconds: 15
-        volumeMounts:
-        - name: secrets
-          mountPath: /etc/kubernetes/secrets
-          readOnly: true
-        - name: ssl-host
-          mountPath: /etc/ssl/certs
-          readOnly: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: secrets
-        secret:
-          secretName: kube-controller-manager
-      - name: ssl-host
-        hostPath:
-          path: /usr/share/ca-certificates
-      dnsPolicy: Default # Don't use cluster DNS.

resources/manifests/kube-dns-deployment.yaml

@@ -1,153 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    addonmanager.kubernetes.io/mode: Reconcile
-spec:
-  # replicas: not specified here:
-  # 1. In order to make Addon Manager do not reconcile this replicas parameter.
-  # 2. Default is 1.
-  # 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
-  strategy:
-    rollingUpdate:
-      maxSurge: 10%
-      maxUnavailable: 0
-  selector:
-    matchLabels:
-      k8s-app: kube-dns
-  template:
-    metadata:
-      labels:
-        k8s-app: kube-dns
-    spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: kube-dns-config
-        configMap:
-          name: kube-dns
-          optional: true
-      containers:
-      - name: kubedns
-        image: ${kubedns_image}
-        resources:
-          # TODO: Set memory limits when we've profiled the container for large
-          # clusters, then set request = limit to keep this container in
-          # guaranteed class. Currently, this container falls into the
-          # "burstable" category so the kubelet doesn't backoff from restarting it.
-          limits:
-            memory: 170Mi
-          requests:
-            cpu: 100m
-            memory: 70Mi
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/kubedns
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        readinessProbe:
-          httpGet:
-            path: /readiness
-            port: 8081
-            scheme: HTTP
-          # we poll on pod startup for the Kubernetes master service and
-          # only setup the /readiness HTTP server once that's available.
-          initialDelaySeconds: 3
-          timeoutSeconds: 5
-        args:
-        - --domain=cluster.local.
-        - --dns-port=10053
-        - --config-dir=/kube-dns-config
-        - --v=2
-        env:
-        - name: PROMETHEUS_PORT
-          value: "10055"
-        ports:
-        - containerPort: 10053
-          name: dns-local
-          protocol: UDP
-        - containerPort: 10053
-          name: dns-tcp-local
-          protocol: TCP
-        - containerPort: 10055
-          name: metrics
-          protocol: TCP
-        volumeMounts:
-        - name: kube-dns-config
-          mountPath: /kube-dns-config
-      - name: dnsmasq
-        image: ${kubedns_dnsmasq_image}
-        livenessProbe:
-          httpGet:
-            path: /healthcheck/dnsmasq
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - -v=2
-        - -logtostderr
-        - -configDir=/etc/k8s/dns/dnsmasq-nanny
-        - -restartDnsmasq=true
-        - --
-        - -k
-        - --cache-size=1000
-        - --no-negcache
-        - --log-facility=-
-        - --server=/cluster.local/127.0.0.1#10053
-        - --server=/in-addr.arpa/127.0.0.1#10053
-        - --server=/ip6.arpa/127.0.0.1#10053
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        # see: https://github.com/kubernetes/kubernetes/issues/29055 for details
-        resources:
-          requests:
-            cpu: 150m
-            memory: 20Mi
-        volumeMounts:
-        - name: kube-dns-config
-          mountPath: /etc/k8s/dns/dnsmasq-nanny
-      - name: sidecar
-        image: ${kubedns_sidecar_image}
-        livenessProbe:
-          httpGet:
-            path: /metrics
-            port: 10054
-            scheme: HTTP
-          initialDelaySeconds: 60
-          timeoutSeconds: 5
-          successThreshold: 1
-          failureThreshold: 5
-        args:
-        - --v=2
-        - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A
-        ports:
-        - containerPort: 10054
-          name: metrics
-          protocol: TCP
-        resources:
-          requests:
-            memory: 20Mi
-            cpu: 10m
-      dnsPolicy: Default  # Don't use cluster DNS.

resources/manifests/kube-dns-svc.yaml

@@ -1,20 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: kube-dns
-  namespace: kube-system
-  labels:
-    k8s-app: kube-dns
-    kubernetes.io/cluster-service: "true"
-    kubernetes.io/name: "KubeDNS"
-spec:
-  selector:
-    k8s-app: kube-dns
-  clusterIP: ${kube_dns_service_ip}
-  ports:
-  - name: dns
-    port: 53
-    protocol: UDP
-  - name: dns-tcp
-    port: 53
-    protocol: TCP

resources/manifests/kube-proxy.yaml

@@ -1,66 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: kube-proxy
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: kube-proxy
-spec:
-  selector:
-    matchLabels:
-      tier: node
-      k8s-app: kube-proxy
-  template:
-    metadata:
-      labels:
-        tier: node
-        k8s-app: kube-proxy
-    spec:
-      containers:
-      - name: kube-proxy
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - proxy
-        - --cluster-cidr=${pod_cidr}
-        - --hostname-override=$(NODE_NAME)
-        - --kubeconfig=/etc/kubernetes/kubeconfig
-        - --proxy-mode=iptables
-        env:
-          - name: NODE_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: spec.nodeName
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /lib/modules
-          name: lib-modules
-          readOnly: true
-        - mountPath: /etc/ssl/certs
-          name: ssl-certs-host
-          readOnly: true
-        - name: kubeconfig
-          mountPath: /etc/kubernetes
-          readOnly: true
-      hostNetwork: true
-      serviceAccountName: kube-proxy
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: lib-modules
-        hostPath:
-          path: /lib/modules
-      - name: ssl-certs-host
-        hostPath:
-          path: /usr/share/ca-certificates
-      - name: kubeconfig
-        secret:
-          secretName: kubeconfig-in-cluster
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/manifests/kube-scheduler-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler

resources/manifests/kube-scheduler.yaml

@@ -1,58 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-scheduler
-spec:
-  replicas: 2
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-scheduler
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-scheduler
-              topologyKey: kubernetes.io/hostname
-      containers:
-      - name: kube-scheduler
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - scheduler
-        - --leader-elect=true
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 10251  # Note: Using default port. Update if --port option is set differently.
-          initialDelaySeconds: 15
-          timeoutSeconds: 15
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule

resources/manifests/kubeconfig-in-cluster.yaml

@@ -1,16 +1,18 @@
 apiVersion: v1
-kind: Secret
+kind: ConfigMap
 metadata:
   name: kubeconfig-in-cluster
   namespace: kube-system
-stringData:
+data:
   kubeconfig: |
     apiVersion: v1
     clusters:
     - name: local
       cluster:
+        # kubeconfig-in-cluster is for control plane components that must reach
+        # kube-apiserver before service IPs are available (e.g.10.3.0.1)
         server: ${server}
-        certificate-authority-data: ${ca_cert}
+        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
     users:
     - name: service-account
       user:

resources/manifests/kubelet-delete-cluster-role-binding.yaml

@@ -1,12 +1,12 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
-  name: system:default-sa
-subjects:
-  - kind: ServiceAccount
-    name: default
-    namespace: kube-system
+  name: kubelet-delete
 roleRef:
-  kind: ClusterRole
-  name: cluster-admin
   apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kubelet-delete
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/kubelet-delete-cluster-role.yaml

@@ -0,0 +1,23 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kubelet-delete
+rules:
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - delete
+  - apiGroups: ["apps"]
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+    verbs:
+      - get
+      - list
+  - apiGroups: [""]
+    resources:
+      - pods/eviction
+    verbs:
+      - create

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: pod-checkpointer
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: pod-checkpointer
-subjects:
-- kind: ServiceAccount
-  name: pod-checkpointer
-  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: pod-checkpointer
-rules:
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["pods"]
-  verbs: ["get", "watch", "list"]
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["secrets", "configmaps"]
-  verbs: ["get"]

resources/manifests/pod-checkpointer.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: pod-checkpointer
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: pod-checkpointer
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: pod-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-    spec:
-      containers:
-      - name: pod-checkpointer
-        image: ${pod_checkpointer_image}
-        command:
-        - /checkpoint
-        - --lock-file=/var/run/lock/pod-checkpointer.lock
-        - --kubeconfig=/etc/checkpointer/kubeconfig
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        imagePullPolicy: Always
-        volumeMounts:
-        - mountPath: /etc/checkpointer
-          name: kubeconfig
-        - mountPath: /etc/kubernetes
-          name: etc-kubernetes
-        - mountPath: /var/run
-          name: var-run
-      serviceAccountName: pod-checkpointer
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      restartPolicy: Always
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: kubeconfig
-        secret:
-          secretName: kubeconfig-in-cluster
-      - name: etc-kubernetes
-        hostPath:
-          path: /etc/kubernetes
-      - name: var-run
-        hostPath:
-          path: /var/run
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/static-manifests/kube-apiserver.yaml

@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-apiserver
+  namespace: kube-system
+  labels:
+    k8s-app: kube-apiserver
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-apiserver
+    image: ${kube_apiserver_image}
+    command:
+    - kube-apiserver
+    - --advertise-address=$(POD_IP)
+    - --allow-privileged=true
+    - --anonymous-auth=false
+    - --authorization-mode=Node,RBAC
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
+    - --enable-admission-plugins=NodeRestriction
+    - --enable-bootstrap-token-auth=true
+    - --etcd-cafile=/etc/kubernetes/pki/etcd-client-ca.crt
+    - --etcd-certfile=/etc/kubernetes/pki/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/pki/etcd-client.key
+    - --etcd-servers=${etcd_servers}
+    - --feature-gates=MutatingAdmissionPolicy=true
+    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.crt
+    - --kubelet-client-key=/etc/kubernetes/pki/apiserver.key
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
+    - --runtime-config=admissionregistration.k8s.io/v1beta1=true,admissionregistration.k8s.io/v1alpha1=true
+    - --secure-port=6443
+    - --service-account-issuer=${service_account_issuer}
+    - --service-account-jwks-uri=${service_account_issuer}/openid/v1/jwks
+    - --service-account-key-file=/etc/kubernetes/pki/service-account.pub
+    - --service-account-signing-key-file=/etc/kubernetes/pki/service-account.key
+    - --service-cluster-ip-range=${service_cidr}
+    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
+    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
+    env:
+    - name: POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    resources:
+      requests:
+        cpu: 150m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl/certs
+      readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki

resources/static-manifests/kube-controller-manager.yaml

@@ -0,0 +1,75 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-controller-manager
+  namespace: kube-system
+  labels:
+    k8s-app: kube-controller-manager
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-controller-manager
+    image: ${kube_controller_manager_image}
+    command:
+    - kube-controller-manager
+    - --authentication-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --allocate-node-cidrs=true
+    - --client-ca-file=/etc/kubernetes/pki/ca.crt
+    - --cluster-cidr=${pod_cidr}
+    - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
+    - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
+    - --cluster-signing-duration=72h
+    - --controllers=*,tokencleaner
+    - --configure-cloud-routes=false
+    - --kubeconfig=/etc/kubernetes/pki/controller-manager.conf
+    - --leader-elect=true
+    - --root-ca-file=/etc/kubernetes/pki/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/pki/service-account.key
+    - --service-cluster-ip-range=${service_cidr}
+    - --use-service-account-credentials=true
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10257
+      initialDelaySeconds: 25
+      timeoutSeconds: 15
+      failureThreshold: 8
+    resources:
+      requests:
+        cpu: 150m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki
+      readOnly: true
+    - name: etc-ssl
+      mountPath: /etc/ssl/certs
+      readOnly: true
+    - name: etc-pki
+      mountPath: /etc/pki
+      readOnly: true
+    - name: flex
+      mountPath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki
+  - name: etc-ssl
+    hostPath:
+      path: /etc/ssl/certs
+  - name: etc-pki
+    hostPath:
+      path: /etc/pki
+  - name: flex
+    hostPath:
+      type: DirectoryOrCreate
+      path: /var/lib/kubelet/volumeplugins

resources/static-manifests/kube-scheduler.yaml

@@ -0,0 +1,44 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: kube-scheduler
+  namespace: kube-system
+  labels:
+    k8s-app: kube-scheduler
+    tier: control-plane
+spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
+    seccompProfile:
+      type: RuntimeDefault
+  containers:
+  - name: kube-scheduler
+    image: ${kube_scheduler_image}
+    command:
+    - kube-scheduler
+    - --authentication-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --authorization-kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --kubeconfig=/etc/kubernetes/pki/scheduler.conf
+    - --leader-elect=true
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10259
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    resources:
+      requests:
+        cpu: 100m
+    volumeMounts:
+    - name: secrets
+      mountPath: /etc/kubernetes/pki/scheduler.conf
+      readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/pki/scheduler.conf

terraform.tfvars.example

@@ -1,6 +1,4 @@
 cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
-asset_dir = "/home/core/mycluster"
 networking = "flannel"
-experimental_self_hosted_etcd = false

tls-aggregation.tf

@@ -0,0 +1,75 @@
+locals {
+  # Kubernetes Aggregation TLS assets map
+  aggregation_tls = var.enable_aggregation ? {
+    "tls/k8s/aggregation-ca.crt"     = tls_self_signed_cert.aggregation-ca[0].cert_pem,
+    "tls/k8s/aggregation-client.crt" = tls_locally_signed_cert.aggregation-client[0].cert_pem,
+    "tls/k8s/aggregation-client.key" = tls_private_key.aggregation-client[0].private_key_pem,
+  } : {}
+}
+
+# Kubernetes Aggregation CA (i.e. front-proxy-ca)
+# Files: tls/{aggregation-ca.crt,aggregation-ca.key}
+
+resource "tls_private_key" "aggregation-ca" {
+  count = var.enable_aggregation ? 1 : 0
+
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_self_signed_cert" "aggregation-ca" {
+  count = var.enable_aggregation ? 1 : 0
+
+  private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
+
+  subject {
+    common_name = "kubernetes-front-proxy-ca"
+  }
+
+  is_ca_certificate     = true
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "cert_signing",
+  ]
+}
+
+# Kubernetes apiserver (i.e. front-proxy-client)
+# Files: tls/{aggregation-client.crt,aggregation-client.key}
+
+resource "tls_private_key" "aggregation-client" {
+  count = var.enable_aggregation ? 1 : 0
+
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "aggregation-client" {
+  count = var.enable_aggregation ? 1 : 0
+
+  private_key_pem = tls_private_key.aggregation-client[0].private_key_pem
+
+  subject {
+    common_name = "kube-apiserver"
+  }
+}
+
+resource "tls_locally_signed_cert" "aggregation-client" {
+  count = var.enable_aggregation ? 1 : 0
+
+  cert_request_pem = tls_cert_request.aggregation-client[0].cert_request_pem
+
+  ca_private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.aggregation-ca[0].cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+

tls-etcd.tf

@@ -1,58 +1,19 @@
-# etcd-client-ca.crt
-resource "local_file" "etcd_client_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
+locals {
+  # etcd TLS assets map
+  etcd_tls = {
+    "tls/etcd/etcd-client-ca.crt" = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/etcd-client.crt"    = tls_locally_signed_cert.client.cert_pem,
+    "tls/etcd/etcd-client.key"    = tls_private_key.client.private_key_pem
+    "tls/etcd/server-ca.crt"      = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/server.crt"         = tls_locally_signed_cert.server.cert_pem
+    "tls/etcd/server.key"         = tls_private_key.server.private_key_pem
+    "tls/etcd/peer-ca.crt"        = tls_self_signed_cert.etcd-ca.cert_pem,
+    "tls/etcd/peer.crt"           = tls_locally_signed_cert.peer.cert_pem
+    "tls/etcd/peer.key"           = tls_private_key.peer.private_key_pem
+  }
 }
 
-# etcd-client.crt
-resource "local_file" "etcd_client_crt" {
-  content  = "${tls_locally_signed_cert.client.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client.crt"
-}
-
-# etcd-client.key
-resource "local_file" "etcd_client_key" {
-  content  = "${tls_private_key.client.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd-client.key"
-}
-
-# server-ca.crt
-resource "local_file" "etcd_server_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
-}
-
-# server.crt
-resource "local_file" "etcd_server_crt" {
-  content  = "${tls_locally_signed_cert.server.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server.crt"
-}
-
-# server.key
-resource "local_file" "etcd_server_key" {
-  content  = "${tls_private_key.server.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd/server.key"
-}
-
-# peer-ca.crt
-resource "local_file" "etcd_peer_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
-}
-
-# peer.crt
-resource "local_file" "etcd_peer_crt" {
-  content  = "${tls_locally_signed_cert.peer.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer.crt"
-}
-
-# peer.key
-resource "local_file" "etcd_peer_key" {
-  content  = "${tls_private_key.peer.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd/peer.key"
-}
-
-# certificates and keys
+# etcd CA
 
 resource "tls_private_key" "etcd-ca" {
   algorithm = "RSA"
@@ -60,8 +21,7 @@ resource "tls_private_key" "etcd-ca" {
 }
 
 resource "tls_self_signed_cert" "etcd-ca" {
-  key_algorithm   = "${tls_private_key.etcd-ca.algorithm}"
-  private_key_pem = "${tls_private_key.etcd-ca.private_key_pem}"
+  private_key_pem = tls_private_key.etcd-ca.private_key_pem
 
   subject {
     common_name  = "etcd-ca"
@@ -78,16 +38,15 @@ resource "tls_self_signed_cert" "etcd-ca" {
   ]
 }
 
-# client certs are used for client (apiserver, locksmith, etcd-operator)
-# to etcd communication
+# etcd Client (apiserver to etcd communication)
+
 resource "tls_private_key" "client" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "client" {
-  key_algorithm   = "${tls_private_key.client.algorithm}"
-  private_key_pem = "${tls_private_key.client.private_key_pem}"
+  private_key_pem = tls_private_key.client.private_key_pem
 
   subject {
     common_name  = "etcd-client"
@@ -96,25 +55,16 @@ resource "tls_cert_request" "client" {
 
   ip_addresses = [
     "127.0.0.1",
-    "${cidrhost(var.service_cidr, 15)}",
-    "${cidrhost(var.service_cidr, 20)}",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "client" {
-  cert_request_pem = "${tls_cert_request.client.cert_request_pem}"
+  cert_request_pem = tls_cert_request.client.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -126,14 +76,15 @@ resource "tls_locally_signed_cert" "client" {
   ]
 }
 
+# etcd Server
+
 resource "tls_private_key" "server" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "server" {
-  key_algorithm   = "${tls_private_key.server.algorithm}"
-  private_key_pem = "${tls_private_key.server.private_key_pem}"
+  private_key_pem = tls_private_key.server.private_key_pem
 
   subject {
     common_name  = "etcd-server"
@@ -142,25 +93,16 @@ resource "tls_cert_request" "server" {
 
   ip_addresses = [
     "127.0.0.1",
-    "${cidrhost(var.service_cidr, 15)}",
-    "${cidrhost(var.service_cidr, 20)}",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "server" {
-  cert_request_pem = "${tls_cert_request.server.cert_request_pem}"
+  cert_request_pem = tls_cert_request.server.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -172,38 +114,29 @@ resource "tls_locally_signed_cert" "server" {
   ]
 }
 
+# etcd Peer
+
 resource "tls_private_key" "peer" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "peer" {
-  key_algorithm   = "${tls_private_key.peer.algorithm}"
-  private_key_pem = "${tls_private_key.peer.private_key_pem}"
+  private_key_pem = tls_private_key.peer.private_key_pem
 
   subject {
     common_name  = "etcd-peer"
     organization = "etcd"
   }
 
-  ip_addresses = [
-    "${cidrhost(var.service_cidr, 20)}",
-  ]
-
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
-    ))}"]
+  dns_names = var.etcd_servers
 }
 
 resource "tls_locally_signed_cert" "peer" {
-  cert_request_pem = "${tls_cert_request.peer.cert_request_pem}"
+  cert_request_pem = tls_cert_request.peer.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -214,3 +147,4 @@ resource "tls_locally_signed_cert" "peer" {
     "client_auth",
   ]
 }
+

tls-k8s.tf

@@ -1,33 +1,28 @@
-# NOTE: Across this module, the following syntax is used at various places:
-#   `"${var.ca_certificate == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key}"`
-#
-# Due to https://github.com/hashicorp/hil/issues/50, both sides of conditions
-# are evaluated, until one of them is discarded. Unfortunately, the
-# `{tls_private_key/tls_self_signed_cert}.kube-ca` resources are created
-# conditionally and might not be present - in which case an error is
-# generated. Because a `count` is used on these ressources, the resources can be
-# referenced as lists with the `.*` notation, and arrays are allowed to be
-# empty. The `join()` interpolation function is then used to cast them back to
-# a string. Since `count` can only be 0 or 1, the returned value is either empty
-# (and discarded anyways) or the desired value.
+locals {
+  # Kubernetes TLS assets map
+  kubernetes_tls = {
+    "tls/k8s/ca.crt"              = tls_self_signed_cert.kube-ca.cert_pem,
+    "tls/k8s/ca.key"              = tls_private_key.kube-ca.private_key_pem,
+    "tls/k8s/apiserver.crt"       = tls_locally_signed_cert.apiserver.cert_pem,
+    "tls/k8s/apiserver.key"       = tls_private_key.apiserver.private_key_pem,
+    "tls/k8s/service-account.pub" = tls_private_key.service-account.public_key_pem
+    "tls/k8s/service-account.key" = tls_private_key.service-account.private_key_pem
+  }
+}
 
 # Kubernetes CA (tls/{ca.crt,ca.key})
-resource "tls_private_key" "kube-ca" {
-  count = "${var.ca_certificate == "" ? 1 : 0}"
 
+resource "tls_private_key" "kube-ca" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_self_signed_cert" "kube-ca" {
-  count = "${var.ca_certificate == "" ? 1 : 0}"
-
-  key_algorithm   = "${tls_private_key.kube-ca.algorithm}"
-  private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
+  private_key_pem = tls_private_key.kube-ca.private_key_pem
 
   subject {
-    common_name  = "kube-ca"
-    organization = "bootkube"
+    common_name  = "kubernetes-ca"
+    organization = "typhoon"
   }
 
   is_ca_certificate     = true
@@ -40,50 +35,39 @@ resource "tls_self_signed_cert" "kube-ca" {
   ]
 }
 
-resource "local_file" "kube-ca-key" {
-  content  = "${var.ca_certificate == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key}"
-  filename = "${var.asset_dir}/tls/ca.key"
-}
-
-resource "local_file" "kube-ca-crt" {
-  content  = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate}"
-  filename = "${var.asset_dir}/tls/ca.crt"
-}
-
 # Kubernetes API Server (tls/{apiserver.key,apiserver.crt})
+
 resource "tls_private_key" "apiserver" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "apiserver" {
-  key_algorithm   = "${tls_private_key.apiserver.algorithm}"
-  private_key_pem = "${tls_private_key.apiserver.private_key_pem}"
+  private_key_pem = tls_private_key.apiserver.private_key_pem
 
   subject {
     common_name  = "kube-apiserver"
-    organization = "kube-master"
+    organization = "system:masters"
   }
 
-  dns_names = [
-    "${var.api_servers}",
+  dns_names = flatten([
+    var.api_servers,
     "kubernetes",
     "kubernetes.default",
     "kubernetes.default.svc",
-    "kubernetes.default.svc.cluster.local",
-  ]
+    "kubernetes.default.svc.${var.cluster_domain_suffix}",
+  ])
 
   ip_addresses = [
-    "${cidrhost(var.service_cidr, 1)}",
+    cidrhost(var.service_cidr, 1),
   ]
 }
 
 resource "tls_locally_signed_cert" "apiserver" {
-  cert_request_pem = "${tls_cert_request.apiserver.cert_request_pem}"
+  cert_request_pem = tls_cert_request.apiserver.cert_request_pem
 
-  ca_key_algorithm   = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.key_algorithm) : var.ca_key_alg}"
-  ca_private_key_pem = "${var.ca_certificate == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key}"
-  ca_cert_pem        = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem): var.ca_certificate}"
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -95,71 +79,101 @@ resource "tls_locally_signed_cert" "apiserver" {
   ]
 }
 
-resource "local_file" "apiserver-key" {
-  content  = "${tls_private_key.apiserver.private_key_pem}"
-  filename = "${var.asset_dir}/tls/apiserver.key"
+# kube-controller-manager
+
+resource "tls_private_key" "controller-manager" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
 }
 
-resource "local_file" "apiserver-crt" {
-  content  = "${tls_locally_signed_cert.apiserver.cert_pem}"
-  filename = "${var.asset_dir}/tls/apiserver.crt"
+resource "tls_cert_request" "controller-manager" {
+  private_key_pem = tls_private_key.controller-manager.private_key_pem
+
+  subject {
+    common_name = "system:kube-controller-manager"
+  }
+}
+
+resource "tls_locally_signed_cert" "controller-manager" {
+  cert_request_pem = tls_cert_request.controller-manager.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+# kube-scheduler
+
+resource "tls_private_key" "scheduler" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "scheduler" {
+  private_key_pem = tls_private_key.scheduler.private_key_pem
+
+  subject {
+    common_name = "system:kube-scheduler"
+  }
+}
+
+resource "tls_locally_signed_cert" "scheduler" {
+  cert_request_pem = tls_cert_request.scheduler.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+# Kubernetes Admin (tls/{admin.key,admin.crt})
+
+resource "tls_private_key" "admin" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "admin" {
+  private_key_pem = tls_private_key.admin.private_key_pem
+
+  subject {
+    common_name  = "kubernetes-admin"
+    organization = "system:masters"
+  }
+}
+
+resource "tls_locally_signed_cert" "admin" {
+  cert_request_pem = tls_cert_request.admin.cert_request_pem
+
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
 }
 
 # Kubernete's Service Account (tls/{service-account.key,service-account.pub})
+
 resource "tls_private_key" "service-account" {
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
-resource "local_file" "service-account-key" {
-  content  = "${tls_private_key.service-account.private_key_pem}"
-  filename = "${var.asset_dir}/tls/service-account.key"
-}
-
-resource "local_file" "service-account-crt" {
-  content  = "${tls_private_key.service-account.public_key_pem}"
-  filename = "${var.asset_dir}/tls/service-account.pub"
-}
-
-# Kubelet
-resource "tls_private_key" "kubelet" {
-  algorithm = "RSA"
-  rsa_bits  = "2048"
-}
-
-resource "tls_cert_request" "kubelet" {
-  key_algorithm   = "${tls_private_key.kubelet.algorithm}"
-  private_key_pem = "${tls_private_key.kubelet.private_key_pem}"
-
-  subject {
-    common_name  = "kubelet"
-    organization = "system:masters"
-  }
-}
-
-resource "tls_locally_signed_cert" "kubelet" {
-  cert_request_pem = "${tls_cert_request.kubelet.cert_request_pem}"
-
-  ca_key_algorithm   = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.key_algorithm) : var.ca_key_alg}"
-  ca_private_key_pem = "${var.ca_certificate == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key}"
-  ca_cert_pem        = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate}"
-
-  validity_period_hours = 8760
-
-  allowed_uses = [
-    "key_encipherment",
-    "digital_signature",
-    "server_auth",
-    "client_auth",
-  ]
-}
-
-resource "local_file" "kubelet-key" {
-  content  = "${tls_private_key.kubelet.private_key_pem}"
-  filename = "${var.asset_dir}/tls/kubelet.key"
-}
-
-resource "local_file" "kubelet-crt" {
-  content  = "${tls_locally_signed_cert.kubelet.cert_pem}"
-  filename = "${var.asset_dir}/tls/kubelet.crt"
-}

variables.tf

@@ -1,96 +1,140 @@
 variable "cluster_name" {
+  type        = string
   description = "Cluster name"
-  type        = "string"
 }
 
 variable "api_servers" {
+  type        = list(string)
   description = "List of URLs used to reach kube-apiserver"
-  type        = "list"
 }
 
 variable "etcd_servers" {
-  description = "List of URLs used to reach etcd servers. Ignored if experimental self-hosted etcd is enabled."
-  type        = "list"
+  type        = list(string)
+  description = "List of URLs used to reach etcd servers."
 }
 
-variable "experimental_self_hosted_etcd" {
-  description = "(Experimental) Create self-hosted etcd assets"
-  default     = false
-}
-
-variable "asset_dir" {
-  description = "Path to a directory where generated assets should be placed (contains secrets)"
-  type        = "string"
-}
-
-variable "cloud_provider" {
-  description = "The provider for cloud services (empty string for no provider)"
-  type        = "string"
-  default     = ""
-}
+# optional
 
 variable "networking" {
-  description = "Choice of networking provider (flannel or calico)"
-  type        = "string"
-  default     = "flannel"
-}
-
-variable "network_mtu" {
-  description = "CNI interface MTU (applies to calico only)"
-  type        = "string"
-  default     = "1500"
-}
-
-variable "pod_cidr" {
-  description = "CIDR IP range to assign Kubernetes pods"
-  type        = "string"
-  default     = "10.2.0.0/16"
-}
-
-variable "service_cidr" {
-  description = <<EOD
-CIDR IP range to assign Kubernetes services.
-The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 20th IP will be reserved for bootstrap self-hosted etcd.
-EOD
-
-  type    = "string"
-  default = "10.3.0.0/24"
-}
-
-variable "container_images" {
-  description = "Container images to use"
-  type        = "map"
-
-  default = {
-    calico            = "quay.io/calico/node:v2.6.3"
-    calico_cni        = "quay.io/calico/cni:v1.11.1"
-    etcd              = "quay.io/coreos/etcd:v3.1.8"
-    etcd_operator     = "quay.io/coreos/etcd-operator:v0.5.0"
-    etcd_checkpointer = "quay.io/coreos/kenc:0.0.2"
-    flannel           = "quay.io/coreos/flannel:v0.9.1-amd64"
-    flannel_cni       = "quay.io/coreos/flannel-cni:v0.3.0"
-    hyperkube         = "gcr.io/google_containers/hyperkube:v1.8.4"
-    kubedns           = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5"
-    kubedns_dnsmasq   = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
-    kubedns_sidecar   = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
-    pod_checkpointer  = "quay.io/coreos/pod-checkpointer:e22cc0e3714378de92f45326474874eb602ca0ac"
+  type        = string
+  description = "Choice of networking provider (flannel or cilium)"
+  default     = "cilium"
+  validation {
+    condition     = contains(["flannel", "cilium"], var.networking)
+    error_message = "networking can be flannel or cilium."
   }
 }
 
-variable "ca_certificate" {
-  description = "Existing PEM-encoded CA certificate (generated if blank)"
-  type        = "string"
-  default     = ""
+variable "pod_cidr" {
+  type        = string
+  description = "CIDR IP range to assign Kubernetes pods"
+  default     = "10.20.0.0/14"
 }
 
-variable "ca_key_alg" {
-  description = "Algorithm used to generate ca_key (required if ca_cert is specified)"
-  type        = "string"
-  default     = "RSA"
+variable "service_cidr" {
+  type        = string
+  description = <<EOD
+CIDR IP range to assign Kubernetes services.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
+EOD
+  default     = "10.3.0.0/24"
 }
 
-variable "ca_private_key" {
-  description = "Existing Certificate Authority private key (required if ca_certificate is set)"
-  type        = "string"
-  default     = ""
+variable "container_images" {
+  type        = map(string)
+  description = "Container images to use"
+  default = {
+    cilium_agent            = "quay.io/cilium/cilium:v1.18.4"
+    cilium_operator         = "quay.io/cilium/operator-generic:v1.18.4"
+    coredns                 = "registry.k8s.io/coredns/coredns:v1.13.1"
+    flannel                 = "docker.io/flannel/flannel:v0.27.0"
+    flannel_cni             = "quay.io/poseidon/flannel-cni:v0.4.2"
+    kube_apiserver          = "registry.k8s.io/kube-apiserver:v1.34.2"
+    kube_controller_manager = "registry.k8s.io/kube-controller-manager:v1.34.2"
+    kube_scheduler          = "registry.k8s.io/kube-scheduler:v1.34.2"
+    kube_proxy              = "registry.k8s.io/kube-proxy:v1.34.2"
+  }
+}
+
+variable "enable_aggregation" {
+  type        = bool
+  description = "Enable the Kubernetes Aggregation Layer (defaults to true)"
+  default     = true
+}
+
+variable "daemonset_tolerations" {
+  type        = list(string)
+  description = "List of additional taint keys kube-system DaemonSets should tolerate (e.g. ['custom-role', 'gpu-role'])"
+  default     = []
+}
+
+# unofficial, temporary, may be removed without notice
+
+variable "external_apiserver_port" {
+  type        = number
+  description = "External kube-apiserver port (e.g. 6443 to match internal kube-apiserver port)"
+  default     = 6443
+}
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by kube-dns"
+  default     = "cluster.local"
+}
+
+variable "components" {
+  description = "Configure pre-installed cluster components"
+  type = object({
+    enable = optional(bool, true)
+    coredns = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    kube_proxy = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    # CNI providers are enabled for pre-install by default, but only the
+    # provider matching var.networking is actually installed.
+    flannel = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+    cilium = optional(
+      object({
+        enable = optional(bool, true)
+      }),
+      {
+        enable = true
+      }
+    )
+  })
+  default = {
+    enable     = true
+    coredns    = null
+    kube_proxy = null
+    flannel    = null
+    cilium     = null
+  }
+  # Set the variable value to the default value when the caller
+  # sets it to null.
+  nullable = false
+}
+
+variable "service_account_issuer" {
+  type        = string
+  description = "kube-apiserver service account token issuer (used as an identifier in 'iss' claims)"
+  default     = "https://kubernetes.default.svc.cluster.local"
 }

versions.tf

@@ -0,0 +1,9 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = ">= 0.13.0, < 2.0.0"
+  required_providers {
+    random = "~> 3.1"
+    tls    = "~> 4.0"
+  }
+}