From b9bef14a0baff7d781836b8fbc936247dec9d60f Mon Sep 17 00:00:00 2001
From: Dalton Hubble <dghubble@gmail.com>
Date: Fri, 5 Apr 2019 00:23:29 -0700
Subject: [PATCH] Add enable_aggregation option (defaults to false)

* Add an `enable_aggregation` variable to enable the kube-apiserver
aggregation layer for adding extension apiservers to clusters
* Aggregation is **disabled** by default. Typhoon recommends you not
enable aggregation. Consider whether less invasive ways to achieve
your goals are possible and whether those goals are well-founded
* Enabling aggregation and extension apiservers increases the attack
surface of a cluster and makes extensions a part of the control plane.
Admins must scrutinize and trust any extension apiserver used.
* Passing a v1.14 CNCF conformance test requires aggregation be enabled.
Having an option for aggregation keeps compliance, but retains the stricter
security posture on default clusters
---
 assets.tf                                     | 18 +++-
 .../manifests/kube-apiserver-secret.yaml      |  4 +
 resources/manifests/kube-apiserver.yaml       |  2 +-
 tls-aggregation.tf                            | 96 +++++++++++++++++++
 variables.tf                                  |  6 ++
 5 files changed, 124 insertions(+), 2 deletions(-)
 create mode 100644 tls-aggregation.tf

diff --git a/assets.tf b/assets.tf
index 38d7442..a1310e8 100644
--- a/assets.tf
+++ b/assets.tf
@@ -38,7 +38,7 @@ resource "template_dir" "manifests" {
     apiserver_port         = "${var.apiserver_port}"
 
     ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
-    ca_key            = "${base64encode(var.ca_private_key == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key)}"
+    ca_key             = "${base64encode(var.ca_private_key == "" ? join(" ", tls_private_key.kube-ca.*.private_key_pem) : var.ca_private_key)}"
     server             = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
     apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
     apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
@@ -48,9 +48,25 @@ resource "template_dir" "manifests" {
     etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
     etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
     etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
+
+    aggregation_flags       = "${var.enable_aggregation == "true" ? indent(8, local.aggregation_flags) : ""}"
+    aggregation_ca_cert     = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""}"
+    aggregation_client_cert = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""}"
+    aggregation_client_key  = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""}"
   }
 }
 
+locals {
+  aggregation_flags = <<EOF
+
+- --proxy-client-cert-file=/etc/kubernetes/secrets/aggregation-client.crt
+- --proxy-client-key-file=/etc/kubernetes/secrets/aggregation-client.key
+- --requestheader-client-ca-file=/etc/kubernetes/secrets/aggregation-ca.crt
+- --requestheader-extra-headers-prefix=X-Remote-Extra-
+- --requestheader-group-headers=X-Remote-Group
+- --requestheader-username-headers=X-Remote-UserEOF
+}
+
 # Generated kubeconfig for Kubelets
 resource "local_file" "kubeconfig-kubelet" {
   content  = "${data.template_file.kubeconfig-kubelet.rendered}"
diff --git a/resources/manifests/kube-apiserver-secret.yaml b/resources/manifests/kube-apiserver-secret.yaml
index 4ead60b..eb0587f 100644
--- a/resources/manifests/kube-apiserver-secret.yaml
+++ b/resources/manifests/kube-apiserver-secret.yaml
@@ -12,3 +12,7 @@ data:
   etcd-client-ca.crt: ${etcd_ca_cert}
   etcd-client.crt: ${etcd_client_cert}
   etcd-client.key: ${etcd_client_key}
+  aggregation-ca.crt: ${aggregation_ca_cert}
+  aggregation-client.crt: ${aggregation_client_cert}
+  aggregation-client.key: ${aggregation_client_key}
+
diff --git a/resources/manifests/kube-apiserver.yaml b/resources/manifests/kube-apiserver.yaml
index c3dfaa3..570c4a7 100644
--- a/resources/manifests/kube-apiserver.yaml
+++ b/resources/manifests/kube-apiserver.yaml
@@ -54,7 +54,7 @@ spec:
         - --insecure-port=0
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
         - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
         - --secure-port=${apiserver_port}
         - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
         - --service-cluster-ip-range=${service_cidr}
diff --git a/tls-aggregation.tf b/tls-aggregation.tf
new file mode 100644
index 0000000..09330ab
--- /dev/null
+++ b/tls-aggregation.tf
@@ -0,0 +1,96 @@
+# Kubernetes Aggregation CA (i.e. front-proxy-ca)
+# Files: tls/{aggregation-ca.crt,aggregation-ca.key}
+
+resource "tls_private_key" "aggregation-ca" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_self_signed_cert" "aggregation-ca" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  key_algorithm   = "${tls_private_key.aggregation-ca.algorithm}"
+  private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
+
+  subject {
+    common_name = "kubernetes-front-proxy-ca"
+  }
+
+  is_ca_certificate     = true
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "cert_signing",
+  ]
+}
+
+resource "local_file" "aggregation-ca-key" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  content  = "${tls_private_key.aggregation-ca.private_key_pem}"
+  filename = "${var.asset_dir}/tls/aggregation-ca.key"
+}
+
+resource "local_file" "aggregation-ca-crt" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  content  = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/aggregation-ca.crt"
+}
+
+# Kubernetes apiserver (i.e. front-proxy-client)
+# Files: tls/{aggregation-client.crt,aggregation-client.key}
+
+resource "tls_private_key" "aggregation-client" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "aggregation-client" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  key_algorithm   = "${tls_private_key.aggregation-client.algorithm}"
+  private_key_pem = "${tls_private_key.aggregation-client.private_key_pem}"
+
+  subject {
+    common_name = "kube-apiserver"
+  }
+}
+
+resource "tls_locally_signed_cert" "aggregation-client" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  cert_request_pem = "${tls_cert_request.aggregation-client.cert_request_pem}"
+
+  ca_key_algorithm   = "${tls_self_signed_cert.aggregation-ca.key_algorithm}"
+  ca_private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
+  ca_cert_pem        = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "client_auth",
+  ]
+}
+
+resource "local_file" "aggregation-client-key" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  content  = "${tls_private_key.aggregation-client.private_key_pem}"
+  filename = "${var.asset_dir}/tls/aggregation-client.key"
+}
+
+resource "local_file" "aggregation-client-crt" {
+  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+
+  content  = "${tls_locally_signed_cert.aggregation-client.cert_pem}"
+  filename = "${var.asset_dir}/tls/aggregation-client.crt"
+}
diff --git a/variables.tf b/variables.tf
index 2381e75..476e9e9 100644
--- a/variables.tf
+++ b/variables.tf
@@ -92,6 +92,12 @@ variable "trusted_certs_dir" {
   default     = "/usr/share/ca-certificates"
 }
 
+variable "enable_aggregation" {
+  description = "Enable the Kubernetes Aggregation Layer (defaults to false, recommended)"
+  type        = "string"
+  default     = "false"
+}
+
 variable "ca_certificate" {
   description = "Existing PEM-encoded CA certificate (generated if blank)"
   type        = "string"