Remove unused raw kubeconfig field outputs

* Remove unused `ca_cert`, `kubelet_cert`, `kubelet_key`,
and `server` outputs
* These outputs were once needed to support clusters with
managed instance groups, but that hasn't been the case for
quite some time

README.md

@@ -1,18 +1,18 @@
-# terraform-render-bootkube
+# terraform-render-bootstrap
 
-`terraform-render-bootkube` is a Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) assets for bootstrapping a Kubernetes cluster.
+`terraform-render-bootstrap` is a Terraform module that renders TLS certificates, static pods, and manifests for bootstrapping a Kubernetes cluster.
 
 ## Audience
 
-`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+`terraform-render-bootstrap` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootstrap module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
-Use the module to declare bootkube assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
+Use the module to declare bootstrap assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
 
 ```hcl
-module "bootkube" {
-  source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=SHA"
+module "bootstrap" {
+  source = "git::https://github.com/poseidon/terraform-render-bootstrap.git?ref=SHA"
 
   cluster_name = "example"
   api_servers = ["node1.example.com"]
@@ -29,21 +29,5 @@ terraform plan
 terraform apply
 ```
 
-Find bootkube assets rendered to the `asset_dir` path. That's it.
+Find bootstrap assets rendered to the `asset_dir` path. That's it.
 
-### Comparison
-
-Render bootkube assets directly with bootkube v0.14.0.
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:6443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
-```
-
-Compare assets. Rendered assets may differ slightly from bootkube assets to reflect decisions made by the [Typhoon](https://github.com/poseidon/typhoon) distribution.
-
-```sh
-pushd /home/core/mycluster
-mv manifests-networking/* manifests
-popd
-diff -rw assets /home/core/mycluster
-```

assets.tf

@@ -1,110 +1,42 @@
-# Self-hosted Kubernetes bootstrap-manifests
-resource "template_dir" "bootstrap-manifests" {
-  source_dir      = "${path.module}/resources/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/bootstrap-manifests"
+# Generated kubeconfig for Kubelets
+data "template_file" "kubeconfig-kubelet" {
+  template = file("${path.module}/resources/kubeconfig-kubelet")
 
-  vars {
-    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-
-    cloud_provider = "${var.cloud_provider}"
-    pod_cidr       = "${var.pod_cidr}"
-    service_cidr   = "${var.service_cidr}"
-
-    trusted_certs_dir = "${var.trusted_certs_dir}"
-    apiserver_port    = "${var.apiserver_port}"
+  vars = {
+    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    kubelet_cert = base64encode(tls_locally_signed_cert.kubelet.cert_pem)
+    kubelet_key  = base64encode(tls_private_key.kubelet.private_key_pem)
+    server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
   }
 }
 
-# Self-hosted Kubernetes manifests
-resource "template_dir" "manifests" {
-  source_dir      = "${path.module}/resources/manifests"
-  destination_dir = "${var.asset_dir}/manifests"
+# Generated admin kubeconfig to bootstrap control plane
+data "template_file" "kubeconfig-admin" {
+  template = file("${path.module}/resources/kubeconfig-admin")
 
-  vars {
-    hyperkube_image        = "${var.container_images["hyperkube"]}"
-    pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}"
-    coredns_image          = "${var.container_images["coredns"]}"
-
-    etcd_servers           = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
-    control_plane_replicas = "${max(2, length(var.etcd_servers))}"
-
-    cloud_provider         = "${var.cloud_provider}"
-    pod_cidr               = "${var.pod_cidr}"
-    service_cidr           = "${var.service_cidr}"
-    cluster_domain_suffix  = "${var.cluster_domain_suffix}"
-    cluster_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
-    trusted_certs_dir      = "${var.trusted_certs_dir}"
-    apiserver_port         = "${var.apiserver_port}"
-
-    ca_cert            = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    ca_key             = "${base64encode(tls_private_key.kube-ca.private_key_pem)}"
-    server             = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-    apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
-    apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
-    serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
-    serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
-
-    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
-
-    aggregation_flags       = "${var.enable_aggregation == "true" ? indent(8, local.aggregation_flags) : ""}"
-    aggregation_ca_cert     = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_self_signed_cert.aggregation-ca.*.cert_pem)) : ""}"
-    aggregation_client_cert = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_locally_signed_cert.aggregation-client.*.cert_pem)) : ""}"
-    aggregation_client_key  = "${var.enable_aggregation == "true" ? base64encode(join(" ", tls_private_key.aggregation-client.*.private_key_pem)) : ""}"
+  vars = {
+    name         = var.cluster_name
+    ca_cert      = base64encode(tls_self_signed_cert.kube-ca.cert_pem)
+    kubelet_cert = base64encode(tls_locally_signed_cert.admin.cert_pem)
+    kubelet_key  = base64encode(tls_private_key.admin.private_key_pem)
+    server       = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
   }
 }
 
-locals {
-  aggregation_flags = <<EOF
-
-- --proxy-client-cert-file=/etc/kubernetes/secrets/aggregation-client.crt
-- --proxy-client-key-file=/etc/kubernetes/secrets/aggregation-client.key
-- --requestheader-client-ca-file=/etc/kubernetes/secrets/aggregation-ca.crt
-- --requestheader-extra-headers-prefix=X-Remote-Extra-
-- --requestheader-group-headers=X-Remote-Group
-- --requestheader-username-headers=X-Remote-UserEOF
-}
-
 # Generated kubeconfig for Kubelets
 resource "local_file" "kubeconfig-kubelet" {
-  content  = "${data.template_file.kubeconfig-kubelet.rendered}"
+  content  = data.template_file.kubeconfig-kubelet.rendered
   filename = "${var.asset_dir}/auth/kubeconfig-kubelet"
 }
 
-# Generated admin kubeconfig (bootkube requires it be at auth/kubeconfig)
-# https://github.com/kubernetes-incubator/bootkube/blob/master/pkg/bootkube/bootkube.go#L42
+# Generated admin kubeconfig to bootstrap control plane
 resource "local_file" "kubeconfig-admin" {
-  content  = "${data.template_file.kubeconfig-admin.rendered}"
+  content  = data.template_file.kubeconfig-admin.rendered
   filename = "${var.asset_dir}/auth/kubeconfig"
 }
 
 # Generated admin kubeconfig in a file named after the cluster
 resource "local_file" "kubeconfig-admin-named" {
-  content  = "${data.template_file.kubeconfig-admin.rendered}"
+  content  = data.template_file.kubeconfig-admin.rendered
   filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
 }
-
-data "template_file" "kubeconfig-kubelet" {
-  template = "${file("${path.module}/resources/kubeconfig-kubelet")}"
-
-  vars {
-    ca_cert      = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-    server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-  }
-}
-
-data "template_file" "kubeconfig-admin" {
-  template = "${file("${path.module}/resources/kubeconfig-admin")}"
-
-  vars {
-    name         = "${var.cluster_name}"
-    ca_cert      = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-    kubelet_cert = "${base64encode(tls_locally_signed_cert.admin.cert_pem)}"
-    kubelet_key  = "${base64encode(tls_private_key.admin.private_key_pem)}"
-    server       = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
-  }
-}

conditional.tf

@@ -1,47 +1,76 @@
 # Assets generated only when certain options are chosen
 
-resource "template_dir" "flannel-manifests" {
-  count           = "${var.networking == "flannel" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/flannel"
-  destination_dir = "${var.asset_dir}/manifests-networking"
+locals {
+  # flannel manifests (manifest.yaml => content)
+  flannel_manifests = {
+    for name in fileset("${path.module}/resources/flannel", "*.yaml"):
+    "manifests-networking/${name}" => templatefile(
+      "${path.module}/resources/flannel/${name}",
+      {
+        flannel_image     = var.container_images["flannel"]
+        flannel_cni_image = var.container_images["flannel_cni"]
+        pod_cidr          = var.pod_cidr
+      }
+    )
+    if var.networking == "flannel"
+  }
 
-  vars {
-    flannel_image     = "${var.container_images["flannel"]}"
-    flannel_cni_image = "${var.container_images["flannel_cni"]}"
+  # calico manifests (manifest.yaml => content)
+  calico_manifests = {
+    for name in fileset("${path.module}/resources/calico", "*.yaml"):
+    "manifests-networking/${name}" => templatefile(
+      "${path.module}/resources/calico/${name}",
+      {
+        calico_image                    = var.container_images["calico"]
+        calico_cni_image                = var.container_images["calico_cni"]
+        network_mtu                     = var.network_mtu
+        network_encapsulation           = indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")
+        ipip_enabled                    = var.network_encapsulation == "ipip" ? true : false
+        ipip_readiness                  = var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""
+        vxlan_enabled                   = var.network_encapsulation == "vxlan" ? true : false
+        network_ip_autodetection_method = var.network_ip_autodetection_method
+        pod_cidr                        = var.pod_cidr
+        enable_reporting                = var.enable_reporting
+      }
+    )
+    if var.networking == "calico"
+  }
 
-    pod_cidr = "${var.pod_cidr}"
+  # kube-router manifests (manifest.yaml => content)
+  kube_router_manifests = {
+    for name in fileset("${path.module}/resources/kube-router", "*.yaml"):
+    "manifests-networking/${name}" => templatefile(
+      "${path.module}/resources/kube-router/${name}",
+      {
+        kube_router_image = var.container_images["kube_router"]
+        flannel_cni_image = var.container_images["flannel_cni"]
+        network_mtu       = var.network_mtu
+      }
+    )
+    if var.networking == "kube-router"
   }
 }
 
-resource "template_dir" "calico-manifests" {
-  count           = "${var.networking == "calico" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/calico"
-  destination_dir = "${var.asset_dir}/manifests-networking"
+# flannel manifests
+resource "local_file" "flannel-manifests" {
+  for_each = local.flannel_manifests
 
-  vars {
-    calico_image     = "${var.container_images["calico"]}"
-    calico_cni_image = "${var.container_images["calico_cni"]}"
-
-    network_mtu                     = "${var.network_mtu}"
-    network_encapsulation           = "${indent(2, var.network_encapsulation == "vxlan" ? "vxlanMode: Always" : "ipipMode: Always")}"
-    ipip_enabled                   = "${var.network_encapsulation == "ipip" ? true : false}"
-    ipip_readiness                 = "${var.network_encapsulation == "ipip" ? indent(16, "- --bird-ready") : ""}"
-    vxlan_enabled                   = "${var.network_encapsulation == "vxlan" ? true : false}"
-    network_ip_autodetection_method = "${var.network_ip_autodetection_method}"
-    pod_cidr                        = "${var.pod_cidr}"
-    enable_reporting                = "${var.enable_reporting}"
-  }
+  filename = "${var.asset_dir}/${each.key}"
+  content  = each.value
 }
 
-resource "template_dir" "kube-router-manifests" {
-  count           = "${var.networking == "kube-router" ? 1 : 0}"
-  source_dir      = "${path.module}/resources/kube-router"
-  destination_dir = "${var.asset_dir}/manifests-networking"
+# Calico manifests
+resource "local_file" "calico-manifests" {
+  for_each = local.calico_manifests
 
-  vars {
-    kube_router_image = "${var.container_images["kube_router"]}"
-    flannel_cni_image = "${var.container_images["flannel_cni"]}"
-
-    network_mtu = "${var.network_mtu}"
-  }
+  filename = "${var.asset_dir}/${each.key}"
+  content  = each.value
+}
+
+# kube-router manifests
+resource "local_file" "kube-router-manifests" {
+  for_each = local.kube_router_manifests
+
+  filename = "${var.asset_dir}/${each.key}"
+  content  = each.value
 }

manifests.tf

@@ -0,0 +1,65 @@
+locals {
+  # Kubernetes static pod manifests (manifest.yaml => content)
+  static_manifests = {
+    for name in fileset("${path.module}/resources/static-manifests", "*.yaml"):
+    "static-manifests/${name}" => templatefile(
+      "${path.module}/resources/static-manifests/${name}",
+      {
+        hyperkube_image   = var.container_images["hyperkube"]
+        etcd_servers      = join(",", formatlist("https://%s:2379", var.etcd_servers))
+        cloud_provider    = var.cloud_provider
+        pod_cidr          = var.pod_cidr
+        service_cidr      = var.service_cidr
+        trusted_certs_dir = var.trusted_certs_dir
+        aggregation_flags = var.enable_aggregation ? indent(4, local.aggregation_flags) : ""
+      }
+    )
+  }
+
+  # Kubernetes control plane manifests (manifest.yaml => content)
+  manifests = {
+    for name in fileset("${path.module}/resources/manifests", "**/*.yaml"):
+    "manifests/${name}" => templatefile(
+      "${path.module}/resources/manifests/${name}",
+      {
+        hyperkube_image        = var.container_images["hyperkube"]
+        coredns_image          = var.container_images["coredns"]
+        control_plane_replicas = max(2, length(var.etcd_servers))
+        pod_cidr               = var.pod_cidr
+        cluster_domain_suffix  = var.cluster_domain_suffix
+        cluster_dns_service_ip = cidrhost(var.service_cidr, 10)
+        trusted_certs_dir      = var.trusted_certs_dir
+        server                 = format("https://%s:%s", var.api_servers[0], var.external_apiserver_port)
+      }
+    )
+  }
+}
+
+# Kubernetes static pod manifests
+resource "local_file" "static-manifests" {
+  for_each = local.static_manifests
+
+  content  = each.value
+  filename = "${var.asset_dir}/${each.key}"
+}
+
+# Kubernetes control plane manifests
+resource "local_file" "manifests" {
+  for_each = local.manifests
+
+  content  = each.value
+  filename = "${var.asset_dir}/${each.key}"
+}
+
+locals {
+  aggregation_flags = <<EOF
+
+- --proxy-client-cert-file=/etc/kubernetes/secrets/aggregation-client.crt
+- --proxy-client-key-file=/etc/kubernetes/secrets/aggregation-client.key
+- --requestheader-client-ca-file=/etc/kubernetes/secrets/aggregation-ca.crt
+- --requestheader-extra-headers-prefix=X-Remote-Extra-
+- --requestheader-group-headers=X-Remote-Group
+- --requestheader-username-headers=X-Remote-User
+EOF
+}
+

outputs.tf

@@ -1,71 +1,44 @@
-output "id" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
-}
-
-output "content_hash" {
-  value = "${sha1("${template_dir.bootstrap-manifests.id} ${template_dir.manifests.id}")}"
-}
 
 output "cluster_dns_service_ip" {
-  value = "${cidrhost(var.service_cidr, 10)}"
+  value = cidrhost(var.service_cidr, 10)
 }
 
 // Generated kubeconfig for Kubelets (i.e. lower privilege than admin)
 output "kubeconfig-kubelet" {
-  value = "${data.template_file.kubeconfig-kubelet.rendered}"
+  value = data.template_file.kubeconfig-kubelet.rendered
 }
 
 // Generated kubeconfig for admins (i.e. human super-user)
 output "kubeconfig-admin" {
-  value = "${data.template_file.kubeconfig-admin.rendered}"
+  value = data.template_file.kubeconfig-admin.rendered
 }
 
 # etcd TLS assets
 
 output "etcd_ca_cert" {
-  value = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  value = tls_self_signed_cert.etcd-ca.cert_pem
 }
 
 output "etcd_client_cert" {
-  value = "${tls_locally_signed_cert.client.cert_pem}"
+  value = tls_locally_signed_cert.client.cert_pem
 }
 
 output "etcd_client_key" {
-  value = "${tls_private_key.client.private_key_pem}"
+  value = tls_private_key.client.private_key_pem
 }
 
 output "etcd_server_cert" {
-  value = "${tls_locally_signed_cert.server.cert_pem}"
+  value = tls_locally_signed_cert.server.cert_pem
 }
 
 output "etcd_server_key" {
-  value = "${tls_private_key.server.private_key_pem}"
+  value = tls_private_key.server.private_key_pem
 }
 
 output "etcd_peer_cert" {
-  value = "${tls_locally_signed_cert.peer.cert_pem}"
+  value = tls_locally_signed_cert.peer.cert_pem
 }
 
 output "etcd_peer_key" {
-  value = "${tls_private_key.peer.private_key_pem}"
-}
-
-# Some platforms may need to reconstruct the kubeconfig directly in user-data.
-# That can't be done with the way template_file interpolates multi-line
-# contents so the raw components of the kubeconfig may be needed.
-
-output "ca_cert" {
-  value = "${base64encode(tls_self_signed_cert.kube-ca.cert_pem)}"
-}
-
-output "kubelet_cert" {
-  value = "${base64encode(tls_locally_signed_cert.kubelet.cert_pem)}"
-}
-
-output "kubelet_key" {
-  value = "${base64encode(tls_private_key.kubelet.private_key_pem)}"
-}
-
-output "server" {
-  value = "${format("https://%s:%s", element(var.api_servers, 0), var.apiserver_port)}"
+  value = tls_private_key.peer.private_key_pem
 }

resources/calico/cluster-role.yaml

@@ -67,18 +67,19 @@ rules:
       - ipamblocks
       - globalnetworkpolicies
       - globalnetworksets
-      - networksets
       - networkpolicies
+      - networksets
       - clusterinformations
       - hostendpoints
+      - blockaffinities
     verbs:
       - get
       - list
       - watch
   - apiGroups: ["crd.projectcalico.org"]
     resources:
-      - felixconfigurations
       - ippools
+      - felixconfigurations
       - clusterinformations
     verbs:
       - create

resources/calico/config.yaml

@@ -36,6 +36,10 @@ data:
           "type": "portmap",
           "snat": true,
           "capabilities": {"portMappings": true}
+        },
+        {
+          "type": "bandwidth",
+          "capabilities": {"bandwidth": true}
         }
       ]
     }

resources/calico/daemonset.yaml

@@ -139,10 +139,10 @@ spec:
             requests:
               cpu: 150m
           livenessProbe:
-            httpGet:
-              path: /liveness
-              port: 9099
-              host: localhost
+            exec:
+              command:
+                - /bin/calico-node
+                - -felix-ready
             periodSeconds: 10
             initialDelaySeconds: 10
             failureThreshold: 6

resources/kube-router/cluster-role.yaml

@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: kube-router

resources/manifests/coredns/config.yaml

@@ -7,7 +7,9 @@ data:
   Corefile: |
     .:53 {
         errors
-        health
+        health {
+          lameduck 5s
+        }
         ready
         log . {
             class error

resources/manifests/coredns/deployment.yaml

@@ -6,7 +6,6 @@ metadata:
   labels:
     k8s-app: coredns
     kubernetes.io/name: "CoreDNS"
-    kubernetes.io/cluster-service: "true"
 spec:
   replicas: ${control_plane_replicas}
   strategy:

resources/manifests/coredns/service-account.yaml

@@ -3,5 +3,3 @@ kind: ServiceAccount
 metadata:
   name: coredns
   namespace: kube-system
-  labels:
-    kubernetes.io/cluster-service: "true"

resources/manifests/coredns/service.yaml

@@ -8,7 +8,6 @@ metadata:
     prometheus.io/port: "9153"
   labels:
     k8s-app: coredns
-    kubernetes.io/cluster-service: "true"
     kubernetes.io/name: "CoreDNS"
 spec:
   selector:

resources/manifests/kube-apiserver-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-apiserver
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: cluster-admin
-subjects:
-- kind: ServiceAccount
-  name: kube-apiserver
-  namespace: kube-system

resources/manifests/kube-apiserver-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-apiserver

resources/manifests/kube-apiserver-secret.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-type: Opaque
-data:
-  apiserver.key: ${apiserver_key}
-  apiserver.crt: ${apiserver_cert}
-  service-account.pub: ${serviceaccount_pub}
-  ca.crt: ${ca_cert}
-  etcd-client-ca.crt: ${etcd_ca_cert}
-  etcd-client.crt: ${etcd_client_cert}
-  etcd-client.key: ${etcd_client_key}
-  aggregation-ca.crt: ${aggregation_ca_cert}
-  aggregation-client.crt: ${aggregation_client_cert}
-  aggregation-client.key: ${aggregation_client_key}
-

resources/manifests/kube-apiserver.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-apiserver
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-apiserver
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-apiserver
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-apiserver
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      serviceAccountName: kube-apiserver
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-apiserver
-        image: ${hyperkube_image}
-        command:
-        - /hyperkube
-        - apiserver
-        - --advertise-address=$(POD_IP)
-        - --allow-privileged=true
-        - --anonymous-auth=false
-        - --authorization-mode=RBAC
-        - --bind-address=0.0.0.0
-        - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --cloud-provider=${cloud_provider}
-        - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority
-        - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
-        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
-        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-        - --etcd-servers=${etcd_servers}
-        - --insecure-port=0
-        - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
-        - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
-        - --secure-port=${apiserver_port}
-        - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
-        - --service-cluster-ip-range=${service_cidr}
-        - --storage-backend=etcd3
-        - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
-        - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
-        env:
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        volumeMounts:
-        - name: secrets
-          mountPath: /etc/kubernetes/secrets
-          readOnly: true
-        - name: ssl-certs-host
-          mountPath: /etc/ssl/certs
-          readOnly: true
-      volumes:
-      - name: secrets
-        secret:
-          secretName: kube-apiserver
-      - name: ssl-certs-host
-        hostPath:
-          path: ${trusted_certs_dir}

resources/manifests/kube-controller-manager-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager

resources/manifests/kube-controller-manager-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-controller-manager
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-controller-manager
-subjects:
-- kind: ServiceAccount
-  name: kube-controller-manager
-  namespace: kube-system

resources/manifests/kube-controller-manager-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-controller-manager

resources/manifests/kube-controller-manager-secret.yaml

@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-type: Opaque
-data:
-  service-account.key: ${serviceaccount_key}
-  ca.crt: ${ca_cert}
-  ca.key: ${ca_key}
-

resources/manifests/kube-controller-manager.yaml

@@ -1,96 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kube-controller-manager
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-controller-manager
-spec:
-  replicas: ${control_plane_replicas}
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-controller-manager
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-controller-manager
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-controller-manager
-              topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      serviceAccountName: kube-controller-manager
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-controller-manager
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - controller-manager
-        - --use-service-account-credentials
-        - --allocate-node-cidrs=true
-        - --cloud-provider=${cloud_provider}
-        - --cluster-cidr=${pod_cidr}
-        - --service-cluster-ip-range=${service_cidr}
-        - --cluster-signing-cert-file=/etc/kubernetes/secrets/ca.crt
-        - --cluster-signing-key-file=/etc/kubernetes/secrets/ca.key
-        - --configure-cloud-routes=false
-        - --leader-elect=true
-        - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
-        - --pod-eviction-timeout=1m
-        - --root-ca-file=/etc/kubernetes/secrets/ca.crt
-        - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
-        livenessProbe:
-          httpGet:
-            scheme: HTTPS
-            path: /healthz
-            port: 10257
-          initialDelaySeconds: 15
-          timeoutSeconds: 15
-        volumeMounts:
-        - name: secrets
-          mountPath: /etc/kubernetes/secrets
-          readOnly: true
-        - name: volumeplugins
-          mountPath: /var/lib/kubelet/volumeplugins
-          readOnly: true          
-        - name: ssl-host
-          mountPath: /etc/ssl/certs
-          readOnly: true
-      volumes:
-      - name: secrets
-        secret:
-          secretName: kube-controller-manager
-      - name: ssl-host
-        hostPath:
-          path: ${trusted_certs_dir}
-      - name: volumeplugins
-        hostPath:
-          path: /var/lib/kubelet/volumeplugins          
-      dnsPolicy: Default # Don't use cluster DNS.

resources/manifests/kube-proxy.yaml

@@ -36,11 +36,11 @@ spec:
         image: ${hyperkube_image}
         command:
         - ./hyperkube
-        - proxy
+        - kube-proxy
         - --cluster-cidr=${pod_cidr}
         - --hostname-override=$(NODE_NAME)
         - --kubeconfig=/etc/kubernetes/kubeconfig
-        - --proxy-mode=iptables
+        - --proxy-mode=ipvs
         env:
           - name: NODE_NAME
             valueFrom:

resources/manifests/kube-scheduler-disruption.yaml

@@ -1,11 +0,0 @@
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-spec:
-  minAvailable: 1
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler

resources/manifests/kube-scheduler-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: kube-scheduler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:kube-scheduler
-subjects:
-- kind: ServiceAccount
-  name: kube-scheduler
-  namespace: kube-system

resources/manifests/kube-scheduler-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: kube-scheduler

resources/manifests/kube-scheduler-volume-scheduler-role-binding.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: volume-scheduler
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: system:volume-scheduler
-subjects:
-- kind: ServiceAccount
-  name: kube-scheduler
-  namespace: kube-system
-

resources/manifests/kube-scheduler.yaml

@@ -1,63 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: kube-scheduler
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-scheduler
-spec:
-  replicas: ${control_plane_replicas}
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-scheduler
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-scheduler
-      annotations:
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: tier
-                  operator: In
-                  values:
-                  - control-plane
-                - key: k8s-app
-                  operator: In
-                  values:
-                  - kube-scheduler
-              topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-cluster-critical
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      serviceAccountName: kube-scheduler
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: kube-scheduler
-        image: ${hyperkube_image}
-        command:
-        - ./hyperkube
-        - scheduler
-        - --leader-elect=true
-        livenessProbe:
-          httpGet:
-            scheme: HTTPS
-            path: /healthz
-            port: 10259
-          initialDelaySeconds: 15
-          timeoutSeconds: 15

resources/manifests/pod-checkpointer-cluster-role-binding.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  name: pod-checkpointer
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: pod-checkpointer
-subjects:
-- kind: ServiceAccount
-  name: pod-checkpointer
-  namespace: kube-system

resources/manifests/pod-checkpointer-cluster-role.yaml

@@ -1,11 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  name: pod-checkpointer
-rules:
-  - apiGroups: [""]
-    resources:
-      - nodes
-      - nodes/proxy
-    verbs:
-      - get

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -1,13 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: pod-checkpointer
-subjects:
-- kind: ServiceAccount
-  name: pod-checkpointer
-  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -1,12 +0,0 @@
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-rules:
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["pods"]
-  verbs: ["get", "watch", "list"]
-- apiGroups: [""] # "" indicates the core API group
-  resources: ["secrets", "configmaps"]
-  verbs: ["get"]

resources/manifests/pod-checkpointer-sa.yaml

@@ -1,5 +0,0 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  namespace: kube-system
-  name: pod-checkpointer

resources/manifests/pod-checkpointer.yaml

@@ -1,72 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: pod-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: pod-checkpointer
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: pod-checkpointer
-  updateStrategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: pod-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-        seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
-    spec:
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      priorityClassName: system-node-critical
-      serviceAccountName: pod-checkpointer
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      containers:
-      - name: pod-checkpointer
-        image: ${pod_checkpointer_image}
-        command:
-        - /checkpoint
-        - --lock-file=/var/run/lock/pod-checkpointer.lock
-        - --kubeconfig=/etc/checkpointer/kubeconfig
-        env:
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /etc/checkpointer
-        - name: etc-kubernetes
-          mountPath: /etc/kubernetes
-        - name: var-run
-          mountPath: /var/run
-      volumes:
-      - name: kubeconfig
-        configMap:
-          name: kubeconfig-in-cluster
-      - name: etc-kubernetes
-        hostPath:
-          path: /etc/kubernetes
-      - name: var-run
-        hostPath:
-          path: /var/run

resources/static-manifests/kube-apiserver.yaml

@@ -1,26 +1,31 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: bootstrap-kube-apiserver
+  name: kube-apiserver
   namespace: kube-system
+  labels:
+    k8s-app: kube-apiserver
+    tier: control-plane
   annotations:
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
   hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
   containers:
   - name: kube-apiserver
     image: ${hyperkube_image}
     command:
     - /hyperkube
-    - apiserver
+    - kube-apiserver
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --anonymous-auth=false
     - --authorization-mode=RBAC
-    - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
     - --cloud-provider=${cloud_provider}
-    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota,Priority
     - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
     - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
     - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
@@ -28,11 +33,10 @@ spec:
     - --insecure-port=0
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
     - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
-    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
-    - --secure-port=${apiserver_port}
+    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname${aggregation_flags}
+    - --secure-port=6443
     - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
     - --service-cluster-ip-range=${service_cidr}
-    - --storage-backend=etcd3
     - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
     - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
     env:
@@ -40,6 +44,9 @@ spec:
       valueFrom:
         fieldRef:
           fieldPath: status.podIP
+    resources:
+      requests:
+        cpu: 200m
     volumeMounts:
     - name: secrets
       mountPath: /etc/kubernetes/secrets

resources/static-manifests/kube-controller-manager.yaml

@@ -1,17 +1,25 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: bootstrap-kube-controller-manager
+  name: kube-controller-manager
   namespace: kube-system
+  labels:
+    k8s-app: kube-controller-manager
+    tier: control-plane
   annotations:
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
   containers:
   - name: kube-controller-manager
     image: ${hyperkube_image}
     command:
-    - ./hyperkube
-    - controller-manager
+    - /hyperkube
+    - kube-controller-manager
     - --allocate-node-cidrs=true
     - --cluster-cidr=${pod_cidr}
     - --service-cluster-ip-range=${service_cidr}
@@ -23,6 +31,17 @@ spec:
     - --leader-elect=true
     - --root-ca-file=/etc/kubernetes/secrets/ca.crt
     - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10257
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    resources:
+      requests:
+        cpu: 200m
     volumeMounts:
     - name: secrets
       mountPath: /etc/kubernetes/secrets
@@ -30,7 +49,6 @@ spec:
     - name: ssl-host
       mountPath: /etc/ssl/certs
       readOnly: true
-  hostNetwork: true
   volumes:
   - name: secrets
     hostPath:

resources/static-manifests/kube-scheduler.yaml

@@ -1,24 +1,42 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: bootstrap-kube-scheduler
+  name: kube-scheduler
   namespace: kube-system
+  labels:
+    k8s-app: kube-scheduler
+    tier: control-plane
   annotations:
     seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
 spec:
+  hostNetwork: true
+  priorityClassName: system-cluster-critical
+  securityContext:
+    runAsNonRoot: true
+    runAsUser: 65534
   containers:
   - name: kube-scheduler
     image: ${hyperkube_image}
     command:
-    - ./hyperkube
-    - scheduler
+    - /hyperkube
+    - kube-scheduler
     - --kubeconfig=/etc/kubernetes/secrets/kubeconfig
     - --leader-elect=true
+    livenessProbe:
+      httpGet:
+        scheme: HTTPS
+        host: 127.0.0.1
+        path: /healthz
+        port: 10259
+      initialDelaySeconds: 15
+      timeoutSeconds: 15
+    resources:
+      requests:
+        cpu: 100m
     volumeMounts:
     - name: secrets
       mountPath: /etc/kubernetes/secrets
       readOnly: true
-  hostNetwork: true
   volumes:
   - name: secrets
     hostPath:

tls-aggregation.tf

@@ -1,27 +1,18 @@
-# NOTE: Across this module, the following workaround is used:
-# `"${var.some_var == "condition" ? join(" ", tls_private_key.aggregation-ca.*.private_key_pem) : ""}"`
-# Due to https://github.com/hashicorp/hil/issues/50, both sides of conditions
-# are evaluated, until one of them is discarded. When a `count` is used resources
-# can be referenced as lists with the `.*` notation, and arrays are allowed to be
-# empty. The `join()` interpolation function is then used to cast them back to
-# a string. Since `count` can only be 0 or 1, the returned value is either empty
-# (and discarded anyways) or the desired value.
-
 # Kubernetes Aggregation CA (i.e. front-proxy-ca)
 # Files: tls/{aggregation-ca.crt,aggregation-ca.key}
 
 resource "tls_private_key" "aggregation-ca" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_self_signed_cert" "aggregation-ca" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = "${tls_private_key.aggregation-ca.algorithm}"
-  private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
+  key_algorithm   = tls_private_key.aggregation-ca[0].algorithm
+  private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
 
   subject {
     common_name = "kubernetes-front-proxy-ca"
@@ -38,16 +29,16 @@ resource "tls_self_signed_cert" "aggregation-ca" {
 }
 
 resource "local_file" "aggregation-ca-key" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  content  = "${tls_private_key.aggregation-ca.private_key_pem}"
+  content  = tls_private_key.aggregation-ca[0].private_key_pem
   filename = "${var.asset_dir}/tls/aggregation-ca.key"
 }
 
 resource "local_file" "aggregation-ca-crt" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  content  = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
+  content  = tls_self_signed_cert.aggregation-ca[0].cert_pem
   filename = "${var.asset_dir}/tls/aggregation-ca.crt"
 }
 
@@ -55,17 +46,17 @@ resource "local_file" "aggregation-ca-crt" {
 # Files: tls/{aggregation-client.crt,aggregation-client.key}
 
 resource "tls_private_key" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
   algorithm = "RSA"
   rsa_bits  = "2048"
 }
 
 resource "tls_cert_request" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  key_algorithm   = "${tls_private_key.aggregation-client.algorithm}"
-  private_key_pem = "${tls_private_key.aggregation-client.private_key_pem}"
+  key_algorithm   = tls_private_key.aggregation-client[0].algorithm
+  private_key_pem = tls_private_key.aggregation-client[0].private_key_pem
 
   subject {
     common_name = "kube-apiserver"
@@ -73,13 +64,13 @@ resource "tls_cert_request" "aggregation-client" {
 }
 
 resource "tls_locally_signed_cert" "aggregation-client" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  cert_request_pem = "${tls_cert_request.aggregation-client.cert_request_pem}"
+  cert_request_pem = tls_cert_request.aggregation-client[0].cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.aggregation-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.aggregation-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.aggregation-ca.cert_pem}"
+  ca_key_algorithm   = tls_self_signed_cert.aggregation-ca[0].key_algorithm
+  ca_private_key_pem = tls_private_key.aggregation-ca[0].private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.aggregation-ca[0].cert_pem
 
   validity_period_hours = 8760
 
@@ -91,15 +82,16 @@ resource "tls_locally_signed_cert" "aggregation-client" {
 }
 
 resource "local_file" "aggregation-client-key" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  content  = "${tls_private_key.aggregation-client.private_key_pem}"
+  content  = tls_private_key.aggregation-client[0].private_key_pem
   filename = "${var.asset_dir}/tls/aggregation-client.key"
 }
 
 resource "local_file" "aggregation-client-crt" {
-  count = "${var.enable_aggregation == "true" ? 1 : 0}"
+  count = var.enable_aggregation ? 1 : 0
 
-  content  = "${tls_locally_signed_cert.aggregation-client.cert_pem}"
+  content  = tls_locally_signed_cert.aggregation-client[0].cert_pem
   filename = "${var.asset_dir}/tls/aggregation-client.crt"
 }
+

tls-etcd.tf

@@ -1,66 +1,66 @@
 # etcd-ca.crt
 resource "local_file" "etcd_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  content  = tls_self_signed_cert.etcd-ca.cert_pem
   filename = "${var.asset_dir}/tls/etcd-ca.crt"
 }
 
 # etcd-ca.key
 resource "local_file" "etcd_ca_key" {
-  content  = "${tls_private_key.etcd-ca.private_key_pem}"
+  content  = tls_private_key.etcd-ca.private_key_pem
   filename = "${var.asset_dir}/tls/etcd-ca.key"
 }
 
 # etcd-client-ca.crt
 resource "local_file" "etcd_client_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  content  = tls_self_signed_cert.etcd-ca.cert_pem
   filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
 }
 
 # etcd-client.crt
 resource "local_file" "etcd_client_crt" {
-  content  = "${tls_locally_signed_cert.client.cert_pem}"
+  content  = tls_locally_signed_cert.client.cert_pem
   filename = "${var.asset_dir}/tls/etcd-client.crt"
 }
 
 # etcd-client.key
 resource "local_file" "etcd_client_key" {
-  content  = "${tls_private_key.client.private_key_pem}"
+  content  = tls_private_key.client.private_key_pem
   filename = "${var.asset_dir}/tls/etcd-client.key"
 }
 
 # server-ca.crt
 resource "local_file" "etcd_server_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  content  = tls_self_signed_cert.etcd-ca.cert_pem
   filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
 }
 
 # server.crt
 resource "local_file" "etcd_server_crt" {
-  content  = "${tls_locally_signed_cert.server.cert_pem}"
+  content  = tls_locally_signed_cert.server.cert_pem
   filename = "${var.asset_dir}/tls/etcd/server.crt"
 }
 
 # server.key
 resource "local_file" "etcd_server_key" {
-  content  = "${tls_private_key.server.private_key_pem}"
+  content  = tls_private_key.server.private_key_pem
   filename = "${var.asset_dir}/tls/etcd/server.key"
 }
 
 # peer-ca.crt
 resource "local_file" "etcd_peer_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  content  = tls_self_signed_cert.etcd-ca.cert_pem
   filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
 }
 
 # peer.crt
 resource "local_file" "etcd_peer_crt" {
-  content  = "${tls_locally_signed_cert.peer.cert_pem}"
+  content  = tls_locally_signed_cert.peer.cert_pem
   filename = "${var.asset_dir}/tls/etcd/peer.crt"
 }
 
 # peer.key
 resource "local_file" "etcd_peer_key" {
-  content  = "${tls_private_key.peer.private_key_pem}"
+  content  = tls_private_key.peer.private_key_pem
   filename = "${var.asset_dir}/tls/etcd/peer.key"
 }
 
@@ -72,8 +72,8 @@ resource "tls_private_key" "etcd-ca" {
 }
 
 resource "tls_self_signed_cert" "etcd-ca" {
-  key_algorithm   = "${tls_private_key.etcd-ca.algorithm}"
-  private_key_pem = "${tls_private_key.etcd-ca.private_key_pem}"
+  key_algorithm   = tls_private_key.etcd-ca.algorithm
+  private_key_pem = tls_private_key.etcd-ca.private_key_pem
 
   subject {
     common_name  = "etcd-ca"
@@ -98,8 +98,8 @@ resource "tls_private_key" "client" {
 }
 
 resource "tls_cert_request" "client" {
-  key_algorithm   = "${tls_private_key.client.algorithm}"
-  private_key_pem = "${tls_private_key.client.private_key_pem}"
+  key_algorithm   = tls_private_key.client.algorithm
+  private_key_pem = tls_private_key.client.private_key_pem
 
   subject {
     common_name  = "etcd-client"
@@ -110,19 +110,15 @@ resource "tls_cert_request" "client" {
     "127.0.0.1",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "client" {
-  cert_request_pem = "${tls_cert_request.client.cert_request_pem}"
+  cert_request_pem = tls_cert_request.client.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -140,8 +136,8 @@ resource "tls_private_key" "server" {
 }
 
 resource "tls_cert_request" "server" {
-  key_algorithm   = "${tls_private_key.server.algorithm}"
-  private_key_pem = "${tls_private_key.server.private_key_pem}"
+  key_algorithm   = tls_private_key.server.algorithm
+  private_key_pem = tls_private_key.server.private_key_pem
 
   subject {
     common_name  = "etcd-server"
@@ -152,19 +148,15 @@ resource "tls_cert_request" "server" {
     "127.0.0.1",
   ]
 
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "localhost",
-    ))}"]
+  dns_names = concat(var.etcd_servers, ["localhost"])
 }
 
 resource "tls_locally_signed_cert" "server" {
-  cert_request_pem = "${tls_cert_request.server.cert_request_pem}"
+  cert_request_pem = tls_cert_request.server.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -182,23 +174,23 @@ resource "tls_private_key" "peer" {
 }
 
 resource "tls_cert_request" "peer" {
-  key_algorithm   = "${tls_private_key.peer.algorithm}"
-  private_key_pem = "${tls_private_key.peer.private_key_pem}"
+  key_algorithm   = tls_private_key.peer.algorithm
+  private_key_pem = tls_private_key.peer.private_key_pem
 
   subject {
     common_name  = "etcd-peer"
     organization = "etcd"
   }
 
-  dns_names = ["${var.etcd_servers}"]
+  dns_names = var.etcd_servers
 }
 
 resource "tls_locally_signed_cert" "peer" {
-  cert_request_pem = "${tls_cert_request.peer.cert_request_pem}"
+  cert_request_pem = tls_cert_request.peer.cert_request_pem
 
-  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
-  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
-  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+  ca_key_algorithm   = tls_self_signed_cert.etcd-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.etcd-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.etcd-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -209,3 +201,4 @@ resource "tls_locally_signed_cert" "peer" {
     "client_auth",
   ]
 }
+

tls-k8s.tf

@@ -6,12 +6,12 @@ resource "tls_private_key" "kube-ca" {
 }
 
 resource "tls_self_signed_cert" "kube-ca" {
-  key_algorithm   = "${tls_private_key.kube-ca.algorithm}"
-  private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
+  key_algorithm   = tls_private_key.kube-ca.algorithm
+  private_key_pem = tls_private_key.kube-ca.private_key_pem
 
   subject {
     common_name  = "kubernetes-ca"
-    organization = "bootkube"
+    organization = "typhoon"
   }
 
   is_ca_certificate     = true
@@ -25,12 +25,12 @@ resource "tls_self_signed_cert" "kube-ca" {
 }
 
 resource "local_file" "kube-ca-key" {
-  content  = "${tls_private_key.kube-ca.private_key_pem}"
+  content  = tls_private_key.kube-ca.private_key_pem
   filename = "${var.asset_dir}/tls/ca.key"
 }
 
 resource "local_file" "kube-ca-crt" {
-  content  = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  content  = tls_self_signed_cert.kube-ca.cert_pem
   filename = "${var.asset_dir}/tls/ca.crt"
 }
 
@@ -42,33 +42,33 @@ resource "tls_private_key" "apiserver" {
 }
 
 resource "tls_cert_request" "apiserver" {
-  key_algorithm   = "${tls_private_key.apiserver.algorithm}"
-  private_key_pem = "${tls_private_key.apiserver.private_key_pem}"
+  key_algorithm   = tls_private_key.apiserver.algorithm
+  private_key_pem = tls_private_key.apiserver.private_key_pem
 
   subject {
     common_name  = "kube-apiserver"
     organization = "system:masters"
   }
 
-  dns_names = [
-    "${var.api_servers}",
+  dns_names = flatten([
+    var.api_servers,
     "kubernetes",
     "kubernetes.default",
     "kubernetes.default.svc",
     "kubernetes.default.svc.${var.cluster_domain_suffix}",
-  ]
+  ])
 
   ip_addresses = [
-    "${cidrhost(var.service_cidr, 1)}",
+    cidrhost(var.service_cidr, 1),
   ]
 }
 
 resource "tls_locally_signed_cert" "apiserver" {
-  cert_request_pem = "${tls_cert_request.apiserver.cert_request_pem}"
+  cert_request_pem = tls_cert_request.apiserver.cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -81,12 +81,12 @@ resource "tls_locally_signed_cert" "apiserver" {
 }
 
 resource "local_file" "apiserver-key" {
-  content  = "${tls_private_key.apiserver.private_key_pem}"
+  content  = tls_private_key.apiserver.private_key_pem
   filename = "${var.asset_dir}/tls/apiserver.key"
 }
 
 resource "local_file" "apiserver-crt" {
-  content  = "${tls_locally_signed_cert.apiserver.cert_pem}"
+  content  = tls_locally_signed_cert.apiserver.cert_pem
   filename = "${var.asset_dir}/tls/apiserver.crt"
 }
 
@@ -98,8 +98,8 @@ resource "tls_private_key" "admin" {
 }
 
 resource "tls_cert_request" "admin" {
-  key_algorithm   = "${tls_private_key.admin.algorithm}"
-  private_key_pem = "${tls_private_key.admin.private_key_pem}"
+  key_algorithm   = tls_private_key.admin.algorithm
+  private_key_pem = tls_private_key.admin.private_key_pem
 
   subject {
     common_name  = "kubernetes-admin"
@@ -108,11 +108,11 @@ resource "tls_cert_request" "admin" {
 }
 
 resource "tls_locally_signed_cert" "admin" {
-  cert_request_pem = "${tls_cert_request.admin.cert_request_pem}"
+  cert_request_pem = tls_cert_request.admin.cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -124,12 +124,12 @@ resource "tls_locally_signed_cert" "admin" {
 }
 
 resource "local_file" "admin-key" {
-  content  = "${tls_private_key.admin.private_key_pem}"
+  content  = tls_private_key.admin.private_key_pem
   filename = "${var.asset_dir}/tls/admin.key"
 }
 
 resource "local_file" "admin-crt" {
-  content  = "${tls_locally_signed_cert.admin.cert_pem}"
+  content  = tls_locally_signed_cert.admin.cert_pem
   filename = "${var.asset_dir}/tls/admin.crt"
 }
 
@@ -141,12 +141,12 @@ resource "tls_private_key" "service-account" {
 }
 
 resource "local_file" "service-account-key" {
-  content  = "${tls_private_key.service-account.private_key_pem}"
+  content  = tls_private_key.service-account.private_key_pem
   filename = "${var.asset_dir}/tls/service-account.key"
 }
 
 resource "local_file" "service-account-crt" {
-  content  = "${tls_private_key.service-account.public_key_pem}"
+  content  = tls_private_key.service-account.public_key_pem
   filename = "${var.asset_dir}/tls/service-account.pub"
 }
 
@@ -158,8 +158,8 @@ resource "tls_private_key" "kubelet" {
 }
 
 resource "tls_cert_request" "kubelet" {
-  key_algorithm   = "${tls_private_key.kubelet.algorithm}"
-  private_key_pem = "${tls_private_key.kubelet.private_key_pem}"
+  key_algorithm   = tls_private_key.kubelet.algorithm
+  private_key_pem = tls_private_key.kubelet.private_key_pem
 
   subject {
     common_name  = "kubelet"
@@ -168,11 +168,11 @@ resource "tls_cert_request" "kubelet" {
 }
 
 resource "tls_locally_signed_cert" "kubelet" {
-  cert_request_pem = "${tls_cert_request.kubelet.cert_request_pem}"
+  cert_request_pem = tls_cert_request.kubelet.cert_request_pem
 
-  ca_key_algorithm   = "${tls_self_signed_cert.kube-ca.key_algorithm}"
-  ca_private_key_pem = "${tls_private_key.kube-ca.private_key_pem}"
-  ca_cert_pem        = "${tls_self_signed_cert.kube-ca.cert_pem}"
+  ca_key_algorithm   = tls_self_signed_cert.kube-ca.key_algorithm
+  ca_private_key_pem = tls_private_key.kube-ca.private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.kube-ca.cert_pem
 
   validity_period_hours = 8760
 
@@ -185,11 +185,12 @@ resource "tls_locally_signed_cert" "kubelet" {
 }
 
 resource "local_file" "kubelet-key" {
-  content  = "${tls_private_key.kubelet.private_key_pem}"
+  content  = tls_private_key.kubelet.private_key_pem
   filename = "${var.asset_dir}/tls/kubelet.key"
 }
 
 resource "local_file" "kubelet-crt" {
-  content  = "${tls_locally_signed_cert.kubelet.cert_pem}"
+  content  = tls_locally_signed_cert.kubelet.cert_pem
   filename = "${var.asset_dir}/tls/kubelet.crt"
 }
+

variables.tf

@@ -1,113 +1,114 @@
 variable "cluster_name" {
+  type        = string
   description = "Cluster name"
-  type        = "string"
 }
 
 variable "api_servers" {
+  type        = list(string)
   description = "List of URLs used to reach kube-apiserver"
-  type        = "list"
 }
 
 variable "etcd_servers" {
+  type        = list(string)
   description = "List of URLs used to reach etcd servers."
-  type        = "list"
 }
 
 variable "asset_dir" {
-  description = "Path to a directory where generated assets should be placed (contains secrets)"
-  type        = "string"
+  type        = string
+  description = "Absolute path to a directory where generated assets should be placed (contains secrets)"
 }
 
 variable "cloud_provider" {
+  type        = string
   description = "The provider for cloud services (empty string for no provider)"
-  type        = "string"
   default     = ""
 }
 
 variable "networking" {
+  type        = string
   description = "Choice of networking provider (flannel or calico or kube-router)"
-  type        = "string"
   default     = "flannel"
 }
 
 variable "network_mtu" {
+  type        = number
   description = "CNI interface MTU (only applies to calico and kube-router)"
-  type        = "string"
-  default     = "1500"
+  default     = 1500
 }
 
 variable "network_encapsulation" {
+  type        = string
   description = "Network encapsulation mode either ipip or vxlan (only applies to calico)"
-  type        = "string"
   default     = "ipip"
 }
 
 variable "network_ip_autodetection_method" {
+  type        = string
   description = "Method to autodetect the host IPv4 address (only applies to calico)"
-  type        = "string"
   default     = "first-found"
 }
 
 variable "pod_cidr" {
+  type        = string
   description = "CIDR IP range to assign Kubernetes pods"
-  type        = "string"
   default     = "10.2.0.0/16"
 }
 
 variable "service_cidr" {
+  type    = string
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
 The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
 EOD
-
-  type    = "string"
   default = "10.3.0.0/24"
 }
 
-variable "cluster_domain_suffix" {
-  description = "Queries for domains with the suffix will be answered by kube-dns"
-  type        = "string"
-  default     = "cluster.local"
-}
 
 variable "container_images" {
+  type        = map(string)
   description = "Container images to use"
-  type        = "map"
 
   default = {
-    calico           = "quay.io/calico/node:v3.7.2"
-    calico_cni       = "quay.io/calico/cni:v3.7.2"
-    flannel          = "quay.io/coreos/flannel:v0.11.0-amd64"
-    flannel_cni      = "quay.io/coreos/flannel-cni:v0.3.0"
-    kube_router      = "cloudnativelabs/kube-router:v0.3.1"
-    hyperkube        = "k8s.gcr.io/hyperkube:v1.14.3"
-    coredns          = "k8s.gcr.io/coredns:1.5.0"
-    pod_checkpointer = "quay.io/coreos/pod-checkpointer:83e25e5968391b9eb342042c435d1b3eeddb2be1"
+    calico      = "quay.io/calico/node:v3.10.1"
+    calico_cni  = "quay.io/calico/cni:v3.10.1"
+    flannel     = "quay.io/coreos/flannel:v0.11.0-amd64"
+    flannel_cni = "quay.io/coreos/flannel-cni:v0.3.0"
+    kube_router = "cloudnativelabs/kube-router:v0.3.2"
+    hyperkube   = "k8s.gcr.io/hyperkube:v1.16.3"
+    coredns     = "k8s.gcr.io/coredns:1.6.5"
   }
 }
 
-variable "enable_reporting" {
-  type        = "string"
-  description = "Enable usage or analytics reporting to upstream component owners (Tigera: Calico)"
-  default     = "false"
-}
 
 variable "trusted_certs_dir" {
+  type        = string
   description = "Path to the directory on cluster nodes where trust TLS certs are kept"
-  type        = "string"
   default     = "/usr/share/ca-certificates"
 }
 
+variable "enable_reporting" {
+  type        = bool
+  description = "Enable usage or analytics reporting to upstream component owners (Tigera: Calico)"
+  default     = false
+}
+
 variable "enable_aggregation" {
+  type        = bool
   description = "Enable the Kubernetes Aggregation Layer (defaults to false, recommended)"
-  type        = "string"
-  default     = "false"
+  default     = false
 }
 
 # unofficial, temporary, may be removed without notice
 
-variable "apiserver_port" {
-  description = "kube-apiserver port"
-  type        = "string"
-  default     = "6443"
+variable "external_apiserver_port" {
+  type        = number
+  description = "External kube-apiserver port (e.g. 6443 to match internal kube-apiserver port)"
+  default     = 6443
 }
+
+variable "cluster_domain_suffix" {
+  type        = string
+  description = "Queries for domains with the suffix will be answered by kube-dns"
+  default     = "cluster.local"
+}
+

versions.tf

@@ -0,0 +1,10 @@
+# Terraform version and plugin versions
+
+terraform {
+  required_version = "~> 0.12.0"
+  required_providers {
+    local    = "~> 1.2"
+    template = "~> 2.1"
+    tls      = "~> 2.0"
+  }
+}