Enable Kubelet TLS bootstrap and NodeRestriction

* Enable bootstrap token authentication on kube-apiserver
* Generate the bootstrap.kubernetes.io/token Secret that
may be used as a bootstrap token
* Generate a bootstrap kubeconfig (with a bootstrap token)
to be securely distributed to nodes. Each Kubelet will use
the bootstrap kubeconfig to authenticate to kube-apiserver
as `system:bootstrappers` and send a node-unique CSR for
kube-controller-manager to automatically approve to issue
a Kubelet certificate and kubeconfig (expires in 72 hours)
* Add ClusterRoleBinding for bootstrap token subjects
(`system:bootstrappers`) to have the `system:node-bootstrapper`
ClusterRole
* Add ClusterRoleBinding for bootstrap token subjects
(`system:bootstrappers`) to have the csr nodeclient ClusterRole
* Add ClusterRoleBinding for bootstrap token subjects
(`system:bootstrappers`) to have the csr selfnodeclient ClusterRole
* Enable NodeRestriction admission controller to limit the
scope of Node or Pod objects a Kubelet can modify to those of
the node itself
* Ability for a Kubelet to delete its Node object is retained
as preemptible nodes or those in auto-scaling instance groups
need to be able to remove themselves on shutdown. This need
continues to have precedence over any risk of a node deleting
itself maliciously

Security notes:

1. Issued Kubelet certificates authenticate as user `system:node:NAME`
and group `system:nodes` and are limited in their authorization
to perform API operations by Node authorization and NodeRestriction
admission. Previously, a Kubelet's authorization was broader. This
is the primary security motivation.

2. The bootstrap kubeconfig credential has the same sensitivity
as the previous generated TLS client-certificate kubeconfig.
It must be distributed securely to nodes. Its compromise still
allows an attacker to obtain a Kubelet kubeconfig

3. Bootstrapping Kubelet kubeconfig's with a limited lifetime offers
a slight security improvement.
  * An attacker who obtains the kubeconfig can likely obtain the
  bootstrap kubeconfig as well, to obtain the ability to renew
  their access
  * A compromised bootstrap kubeconfig could plausibly be handled
  by replacing the bootstrap token Secret, distributing the token
  to new nodes, and expiration. Whereas a compromised TLS-client
  certificate kubeconfig can't be revoked (no CRL). However,
  replacing a bootstrap token can be impractical in real cluster
  environments, so the limited lifetime is mostly a theoretical
  benefit.
  * Cluster CSR objects are visible via kubectl which is nice

4. Bootstrapping node-unique Kubelet kubeconfigs means Kubelet
clients have more identity information, which can improve the
utility of audits and future features

Rel: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet-tls-bootstrapping/

auth.tf

@@ -5,18 +5,33 @@ locals {
   }
 }
 
-# Generated kubeconfig for Kubelets
-data "template_file" "kubeconfig-kubelet" {
-  template = file("${path.module}/resources/kubeconfig-kubelet")
+# Generate a cryptographically random token id (public)
+resource random_string "bootstrap-token-id" {
+  length  = 6
+  upper   = false
+  special = false
+}
+
+# Generate a cryptographically random token secret
+resource random_string "bootstrap-token-secret" {
+  length  = 16
+  upper   = false
+  special = false
+}
+
+# Generated kubeconfig to bootstrap Kubelets
+data "template_file" "kubeconfig-bootstrap" {
+  template = file("${path.module}/resources/kubeconfig-bootstrap")
 
   vars = {
     ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
-    kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem)
-    kubelet_key  = base64encode(tls_private_key.kubelet.private_key_pem)
     server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+    token_id     = random_string.bootstrap-token-id.result
+    token_secret = random_string.bootstrap-token-secret.result
   }
 }
 
+
 # Generated admin kubeconfig to bootstrap control plane
 data "template_file" "kubeconfig-admin" {
   template = file("${path.module}/resources/kubeconfig-admin")
@@ -30,14 +45,6 @@ data "template_file" "kubeconfig-admin" {
   }
 }
 
-# Generated kubeconfig for Kubelets
-resource "local_file" "kubeconfig-kubelet" {
-  count = var.asset_dir == "" ? 0 : 1
-
-  content  = data.template_file.kubeconfig-kubelet.rendered
-  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
-}
-
 # Generated admin kubeconfig to bootstrap control plane
 resource "local_file" "kubeconfig-admin" {
   count = var.asset_dir == "" ? 0 : 1

manifests.tf

@@ -36,6 +36,8 @@ locals {
         trusted_certs_dir      = var.trusted_certs_dir
         server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
         daemonset_tolerations  = var.daemonset_tolerations
+        token_id               = random_string.bootstrap-token-id.result
+        token_secret           = random_string.bootstrap-token-secret.result
       }
     )
   }

outputs.tf

@@ -5,7 +5,7 @@ output "cluster_dns_service_ip" {
 
 // Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
 output "kubeconfig-kubelet" {
-  value     = data.template_file.kubeconfig-kubelet.rendered
+  value     = data.template_file.kubeconfig-bootstrap.rendered
   sensitive = true
 }
 

resources/kubeconfig-bootstrap

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: kubelet
+  user:
+    token: ${token_id}.${token_secret}
+contexts:
+- context:
+    cluster: local
+    user: kubelet

resources/manifests/bootstrap-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Bind system:bootstrappers to ClusterRole for node bootstrap
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-bootstrapper
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-new-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve new CSRs from "system:bootstrappers" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-new
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:bootstrappers

resources/manifests/bootstrap-renew-approve-cluster-role-binding.yaml

@@ -0,0 +1,13 @@
+# Approve renewal CSRs from "system:nodes" subjects
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: bootstrap-approve-renew
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:nodes

resources/manifests/bootstrap-token.yaml

@@ -0,0 +1,12 @@
+apiVersion: v1
+kind: Secret
+type: bootstrap.kubernetes.io/token
+metadata:
+  # Name MUST be of form "bootstrap-token-<token_id>"
+  name: bootstrap-token-${token_id}
+  namespace: kube-system
+stringData:
+  description: "Typhoon generated bootstrap token"
+  token-id: ${token_id}
+  token-secret: ${token_secret}
+  usage-bootstrap-authentication: "true"

resources/manifests/kubelet-delete-cluster-role-binding.yaml

@@ -7,6 +7,6 @@ roleRef:
   kind: ClusterRole
   name: kubelet-delete
 subjects:
-- kind: Group
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
   name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/manifests/kubelet-nodes-cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: system-nodes
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:node
-subjects:
-- kind: Group
-  name: system:nodes
-  apiGroup: rbac.authorization.k8s.io

resources/static-manifests/kube-apiserver.yaml

@@ -22,8 +22,10 @@ spec:
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --anonymous-auth=false
-    - --authorization-mode=RBAC
+    - --authorization-mode=Node,RBAC
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --enable-admission-plugins=NodeRestriction
+    - --enable-bootstrap-token-auth=true
     - --cloud-provider=${cloud_provider}
     - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
     - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt

resources/static-manifests/kube-controller-manager.yaml

@@ -25,6 +25,7 @@ spec:
     - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
     - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
     - --configure-cloud-routes=false
+    - --experimental-cluster-signing-duration=72h
     - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
     - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
     - --leader-elect=true

versions.tf

@@ -4,6 +4,7 @@ terraform {
   required_version = "~> 0.12.0"
   required_providers {
     local    = "~> 1.2"
+    random   = "~> 2.2"
     template = "~> 2.1"
     tls      = "~> 2.0"
   }