Add admin kubeconfig and limit Kubelet cert to system:nodes group

* Change Kubelet TLS client certificate to belong to the system:nodes
group instead of the system:masters group (more limited)
* Bind the system:node ClusterRole to the system:nodes group (yes,
the ClusterRole is singular)
* Generate separate admin.crt and admin.key files (which do still use
system:masters). Output kubeconfig-kubelet and kubeconfig-admin values
from the module
* Remove the kubeconfig output to force users to pick the correct
kubeconfig, depending on how the output is used (action required!)

Related:

* https://kubernetes.io/docs/reference/access-authn-authz/rbac/#core-component-roles

Note, NodeAuthorizer/NodeRestriction would be an enhancement, but to
work across platforms it effectively requires TLS bootstraping which
doesn't have a viable attestation strategy and clashes with CCM. This
change improves Kubelet limitations, but intentionally doesn't aim to
steer toward NodeAuthorizer/NodeRestriction

assets.tf

@@ -50,20 +50,27 @@ resource "template_dir" "manifests" {
   }
 }
 
-# Generated kubeconfig
-resource "local_file" "kubeconfig" {
-  content  = "${data.template_file.kubeconfig.rendered}"
+# Generated kubeconfig for Kubelets
+resource "local_file" "kubeconfig-kubelet" {
+  content  = "${data.template_file.kubeconfig-kubelet.rendered}"
+  filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
+}
+
+# Generated admin kubeconfig (bootkube requires it be at auth/kubeconfig)
+# https://github.com/kubernetes-incubator/bootkube/blob/master/pkg/bootkube/bootkube.go#L42
+resource "local_file" "kubeconfig-admin" {
+  content  = "${data.template_file.kubeconfig-admin.rendered}"
   filename = "${var.asset_dir}/auth/kubeconfig"
 }
 
-# Generated kubeconfig with user-context
-resource "local_file" "user-kubeconfig" {
-  content  = "${data.template_file.user-kubeconfig.rendered}"
+# Generated admin kubeconfig with a named context
+resource "local_file" "kubeconfig-admin-context" {
+  content  = "${data.template_file.kubeconfig-admin-context.rendered}"
   filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
 }
 
-data "template_file" "kubeconfig" {
-  template = "${file("${path.module}/resources/kubeconfig")}"
+data "template_file" "kubeconfig-kubelet" {
+  template = "${file("${path.module}/resources/kubeconfig-kubelet")}"
 
   vars {
     ca_cert      = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
@@ -73,14 +80,25 @@ data "template_file" "kubeconfig" {
   }
 }
 
-data "template_file" "user-kubeconfig" {
-  template = "${file("${path.module}/resources/user-kubeconfig")}"
+data "template_file" "kubeconfig-admin" {
+  template = "${file("${path.module}/resources/kubeconfig-admin")}"
+
+  vars {
+    ca_cert      = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
+    kubelet_cert = "${base64encode(tls_locally_signed_cert.admin.cert_pem)}"
+    kubelet_key  = "${base64encode(tls_private_key.admin.private_key_pem)}"
+    server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
+  }
+}
+
+data "template_file" "kubeconfig-admin-context" {
+  template = "${file("${path.module}/resources/kubeconfig-admin-context")}"
 
   vars {
     name         = "${var.cluster_name}"
     ca_cert      = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
+    kubelet_cert = "${base64encode(tls_locally_signed_cert.admin.cert_pem)}"
+    kubelet_key  = "${base64encode(tls_private_key.admin.private_key_pem)}"
     server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
   }
 }

outputs.tf

@@ -1,5 +1,5 @@
 output "id" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${local_file.kubeconfig.id}")}"
+  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
 }
 
 output "content_hash" {
@@ -15,12 +15,29 @@ output "cluster_dns_service_ip" {
   value = "${cidrhost(var.service_cidr, 10)}"
 }
 
-output "kubeconfig" {
-  value = "${data.template_file.kubeconfig.rendered}"
+// Intentionally Removed! (kubelets should now use kubeconfig-kubelet)
+// output "kubeconfig" {
+//  value = "${data.template_file.kubeconfig.rendered}"
+// }
+
+// Deprecated (use kubeconfig-admin-context)
+output "user-kubeconfig" {
+  value = "${data.template_file.kubeconfig-admin-context.rendered}"
 }
 
-output "user-kubeconfig" {
-  value = "${data.template_file.user-kubeconfig.rendered}"
+// Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
+output "kubeconfig-kubelet" {
+  value = "${data.template_file.kubeconfig-kubelet.rendered}"
+}
+
+// Generated kubeconfig for admins (i.e. human super-user)
+output "kubeconfig-admin" {
+  value = "${data.template_file.kubeconfig-admin.rendered}"
+}
+
+// Generated kubeconfig for admins with a context
+output "kubeconfig-admin-context" {
+  value = "${data.template_file.kubeconfig-admin-context.rendered}"
 }
 
 # etcd TLS assets

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -17,22 +17,22 @@ spec:
     - --service-cluster-ip-range=${service_cidr}
     - --cloud-provider=${cloud_provider}
     - --configure-cloud-routes=false
-    - --kubeconfig=/etc/kubernetes/kubeconfig
+    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
     - --leader-elect=true
-    - --root-ca-file=/etc/kubernetes/bootstrap-secrets/ca.crt
-    - --service-account-private-key-file=/etc/kubernetes/bootstrap-secrets/service-account.key
+    - --root-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
     volumeMounts:
-    - name: kubernetes
-      mountPath: /etc/kubernetes
+    - name: secrets
+      mountPath: /etc/kubernetes/secrets
       readOnly: true
     - name: ssl-host
       mountPath: /etc/ssl/certs
       readOnly: true
   hostNetwork: true
   volumes:
-  - name: kubernetes
+  - name: secrets
     hostPath:
-      path: /etc/kubernetes
+      path: /etc/kubernetes/bootstrap-secrets
   - name: ssl-host
     hostPath:
       path: ${trusted_certs_dir}

resources/bootstrap-manifests/bootstrap-scheduler.yaml

@@ -12,14 +12,14 @@ spec:
     command:
     - ./hyperkube
     - scheduler
-    - --kubeconfig=/etc/kubernetes/kubeconfig
+    - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
     - --leader-elect=true
     volumeMounts:
-    - name: kubernetes
-      mountPath: /etc/kubernetes
+    - name: secrets
+      mountPath: /etc/kubernetes/secrets
       readOnly: true
   hostNetwork: true
   volumes:
-  - name: kubernetes
+  - name: secrets
     hostPath:
-      path: /etc/kubernetes
+      path: /etc/kubernetes/bootstrap-secrets

resources/kubeconfig-admin

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: local
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: admin
+  user:
+    client-certificate-data: ${kubelet_cert}
+    client-key-data: ${kubelet_key}
+contexts:
+- context:
+    cluster: local
+    user: admin

resources/kubeconfig-admin-context

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: ${name}-cluster
+  cluster:
+    server: ${server}
+    certificate-authority-data: ${ca_cert}
+users:
+- name: ${name}-user
+  user:
+    client-certificate-data: ${kubelet_cert}
+    client-key-data: ${kubelet_key}
+contexts:
+- name: ${name}-context
+  context:
+    cluster: ${name}-cluster
+    user: ${name}-user

resources/manifests/kubelet-nodes-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: system-nodes
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node
+subjects:
+- kind: Group
+  name: system:nodes
+  apiGroup: rbac.authorization.k8s.io

tls-k8s.tf

@@ -51,6 +51,7 @@ resource "local_file" "kube-ca-crt" {
 }
 
 # Kubernetes API Server (tls/{apiserver.key,apiserver.crt})
+
 resource "tls_private_key" "apiserver" {
   algorithm = "RSA"
   rsa_bits  = "2048"
@@ -105,7 +106,51 @@ resource "local_file" "apiserver-crt" {
   filename = "${var.asset_dir}/tls/apiserver.crt"
 }
 
+# Kubernetes Admin (tls/{admin.key,admin.crt})
+
+resource "tls_private_key" "admin" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "admin" {
+  key_algorithm   = "${tls_private_key.admin.algorithm}"
+  private_key_pem = "${tls_private_key.admin.private_key_pem}"
+
+  subject {
+    common_name  = "kubernetes-admin"
+    organization = "system:masters"
+  }
+}
+
+resource "tls_locally_signed_cert" "admin" {
+  cert_request_pem = "${tls_cert_request.admin.cert_request_pem}"
+
+  ca_key_algorithm   = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.key_algorithm) : var.ca_key_alg}"
+  ca_private_key_pem = "${var.ca_certificate == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key}"
+  ca_cert_pem        = "${var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem): var.ca_certificate}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+resource "local_file" "admin-key" {
+  content  = "${tls_private_key.admin.private_key_pem}"
+  filename = "${var.asset_dir}/tls/admin.key"
+}
+
+resource "local_file" "admin-crt" {
+  content  = "${tls_locally_signed_cert.admin.cert_pem}"
+  filename = "${var.asset_dir}/tls/admin.crt"
+}
+
 # Kubernete's Service Account (tls/{service-account.key,service-account.pub})
+
 resource "tls_private_key" "service-account" {
   algorithm = "RSA"
   rsa_bits  = "2048"
@@ -122,6 +167,7 @@ resource "local_file" "service-account-crt" {
 }
 
 # Kubelet
+
 resource "tls_private_key" "kubelet" {
   algorithm = "RSA"
   rsa_bits  = "2048"
@@ -133,7 +179,7 @@ resource "tls_cert_request" "kubelet" {
 
   subject {
     common_name  = "kubelet"
-    organization = "system:masters"
+    organization = "system:nodes"
   }
 }