Update README to correspond to bootkube v0.12.0

* Remove flags deprecated in Kubernetes v1.10.x
* https://github.com/poseidon/terraform-render-bootkube/pull/50

.gitignore

@@ -1,2 +1,4 @@
 *.tfvars
+.terraform
 *.tfstate*
+assets

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Dalton Hubble
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -1,58 +1,50 @@
-# bootkube-terraform
+# terraform-render-bootkube
 
-`bootkube-terraform` is a Terraform module that renders [bootkube](https://github.com/kubernetes-incubator/bootkube) assets, just like running the binary `bootkube render`. It aims to provide the same variable names, defaults, features, and outputs.
+`terraform-render-bootkube` is a Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) assets for bootstrapping a Kubernetes cluster.
+
+## Audience
+
+`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
-Use the `bootkube-terraform` module within your existing Terraform configs. Provide the variables listed in `variables.tf` or check `terraform.tfvars.example` for examples.
+Use the module to declare bootkube assets. Check [variables.tf](variables.tf) for options and [terraform.tfvars.example](terraform.tfvars.example) for examples.
 
 ```hcl
 module "bootkube" {
-  source = "git://https://github.com/dghubble/bootkube-terraform.git"
+  source = "git://https://github.com/poseidon/terraform-render-bootkube.git?ref=SHA"
 
   cluster_name = "example"
   api_servers = ["node1.example.com"]
-  etcd_servers = ["http://127.0.0.1:2379"]
+  etcd_servers = ["node1.example.com"]
   asset_dir = "/home/core/clusters/mycluster"
-  experimental_self_hosted_etcd = false
 }
 ```
 
-Alternately, use a local checkout of this repo and copy `terraform.tfvars.example` to `terraform.tfvars` to generate assets without an existing terraform config repo.
-
-Generate the bootkube assets.
+Generate the assets.
 
 ```sh
-terraform get
+terraform init
+terraform get --update
 terraform plan
 terraform apply
 ```
 
+Find bootkube assets rendered to the `asset_dir` path. That's it.
+
 ### Comparison
 
-Render bootkube assets directly with bootkube v0.4.2.
-
-#### On-host etcd
+Render bootkube assets directly with bootkube v0.12.0.
 
 ```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=http://127.0.0.1:2379
+bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
 ```
 
-Compare assets. The only diffs you should see are TLS credentials.
+Compare assets. Rendered assets may differ slightly from bootkube assets to reflect decisions made by the [Typhoon](https://github.com/poseidon/typhoon) distribution.
 
 ```sh
-diff -rw assets /home/core/cluster/mycluster
+pushd /home/core/mycluster
+mv manifests-networking/* manifests
+popd
+diff -rw assets /home/core/mycluster
 ```
-
-#### Self-hosted etcd
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd
-```
-
-Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment.
-
-```sh
-diff -rw assets /home/core/cluster/mycluster
-```
-

assets.tf

@@ -1,15 +1,17 @@
-# Self-hosted Kubernetes bootstrap manifests
+# Self-hosted Kubernetes bootstrap-manifests
 resource "template_dir" "bootstrap-manifests" {
   source_dir      = "${path.module}/resources/bootstrap-manifests"
   destination_dir = "${var.asset_dir}/bootstrap-manifests"
 
   vars {
     hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379,http://127.0.0.1:12379", cidrhost(var.service_cidr, 15)) : join(",", var.etcd_servers)}"
+    etcd_servers    = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
 
     cloud_provider = "${var.cloud_provider}"
     pod_cidr       = "${var.pod_cidr}"
     service_cidr   = "${var.service_cidr}"
+
+    trusted_certs_dir = "${var.trusted_certs_dir}"
   }
 }
 
@@ -19,25 +21,46 @@ resource "template_dir" "manifests" {
   destination_dir = "${var.asset_dir}/manifests"
 
   vars {
-    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", var.etcd_servers)}"
+    hyperkube_image        = "${var.container_images["hyperkube"]}"
+    pod_checkpointer_image = "${var.container_images["pod_checkpointer"]}"
+    kubedns_image          = "${var.container_images["kubedns"]}"
+    kubedns_dnsmasq_image  = "${var.container_images["kubedns_dnsmasq"]}"
+    kubedns_sidecar_image  = "${var.container_images["kubedns_sidecar"]}"
 
-    cloud_provider = "${var.cloud_provider}"
-    
-    pod_cidr       = "${var.pod_cidr}"
-    service_cidr   = "${var.service_cidr}"
-    kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
+    etcd_servers = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
+
+    cloud_provider        = "${var.cloud_provider}"
+    pod_cidr              = "${var.pod_cidr}"
+    service_cidr          = "${var.service_cidr}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
+    kube_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
+    trusted_certs_dir     = "${var.trusted_certs_dir}"
 
     ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
+    server             = "${format("https://%s:443", element(var.api_servers, 0))}"
     apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
     apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
     serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
     serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
+
+    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
+    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
   }
 }
 
+# Generated kubeconfig
+resource "local_file" "kubeconfig" {
+  content  = "${data.template_file.kubeconfig.rendered}"
+  filename = "${var.asset_dir}/auth/kubeconfig"
+}
+
+# Generated kubeconfig with user-context
+resource "local_file" "user-kubeconfig" {
+  content  = "${data.template_file.user-kubeconfig.rendered}"
+  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
+}
 
-# Generated kubeconfig (auth/kubeconfig)
 data "template_file" "kubeconfig" {
   template = "${file("${path.module}/resources/kubeconfig")}"
 
@@ -49,12 +72,6 @@ data "template_file" "kubeconfig" {
   }
 }
 
-resource "local_file" "kubeconfig" {
-  content  = "${data.template_file.kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/kubeconfig"
-}
-
-# Generated kubeconfig (auth/kubeconfig)
 data "template_file" "user-kubeconfig" {
   template = "${file("${path.module}/resources/user-kubeconfig")}"
 
@@ -66,8 +83,3 @@ data "template_file" "user-kubeconfig" {
     server       = "${format("https://%s:443", element(var.api_servers, 0))}"
   }
 }
-
-resource "local_file" "user-kubeconfig" {
-  content  = "${data.template_file.user-kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
-}

conditional.tf

@@ -0,0 +1,28 @@
+# Assets generated only when certain options are chosen
+
+resource "template_dir" "flannel-manifests" {
+  count           = "${var.networking == "flannel" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/flannel"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    flannel_image     = "${var.container_images["flannel"]}"
+    flannel_cni_image = "${var.container_images["flannel_cni"]}"
+
+    pod_cidr = "${var.pod_cidr}"
+  }
+}
+
+resource "template_dir" "calico-manifests" {
+  count           = "${var.networking == "calico" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/calico"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    calico_image     = "${var.container_images["calico"]}"
+    calico_cni_image = "${var.container_images["calico_cni"]}"
+
+    network_mtu = "${var.network_mtu}"
+    pod_cidr    = "${var.pod_cidr}"
+  }
+}

etcd-assets.tf

@@ -1,68 +0,0 @@
-# Experimental self-hosted etcd
-
-# etcd pod and service bootstrap-manifests
-
-data "template_file" "bootstrap-etcd" {
-  template = "${file("${path.module}/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml")}"
-  vars {
-    etcd_image = "${var.container_images["etcd"]}"
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
-  }
-}
-
-resource "local_file" "bootstrap-etcd" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.bootstrap-etcd.rendered}"
-  filename = "${var.asset_dir}/experimental/bootstrap-manifests/bootstrap-etcd.yaml"
-}
-
-data "template_file" "bootstrap-etcd-service" {
-  template = "${file("${path.module}/resources/etcd/bootstrap-etcd-service.json")}"
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
-  }
-}
-
-resource "local_file" "bootstrap-etcd-service" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.bootstrap-etcd-service.rendered}"
-  filename = "${var.asset_dir}/etcd/bootstrap-etcd-service.json"
-}
-
-data "template_file" "etcd-tpr" {
-  template = "${file("${path.module}/resources/etcd/migrate-etcd-cluster.json")}"
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
-  }
-}
-
-resource "local_file" "etcd-tpr" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.etcd-tpr.rendered}"
-  filename = "${var.asset_dir}/etcd/migrate-etcd-cluster.json"
-}
-
-# etcd operator deployment and service manifests
-
-resource "local_file" "etcd-operator" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  depends_on = ["template_dir.manifests"]
-
-  content  = "${file("${path.module}/resources/experimental/manifests/etcd-operator.yaml")}"
-  filename = "${var.asset_dir}/experimental/manifests/etcd-operator.yaml"
-}
-
-data "template_file" "etcd-service" {
-  template = "${file("${path.module}/resources/experimental/manifests/etcd-service.yaml")}"
-  vars {
-    etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
-  }
-}
-
-resource "local_file" "etcd-service" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  depends_on = ["template_dir.manifests"]
-
-  content  = "${data.template_file.etcd-service.rendered}"
-  filename = "${var.asset_dir}/experimental/manifests/etcd-service.yaml"
-}

outputs.tf

@@ -10,16 +10,42 @@ output "kube_dns_service_ip" {
   value = "${cidrhost(var.service_cidr, 10)}"
 }
 
-output "etcd_service_ip" {
-  value = "${cidrhost(var.service_cidr, 15)}"
-}
-
 output "kubeconfig" {
   value = "${data.template_file.kubeconfig.rendered}"
 }
 
 output "user-kubeconfig" {
-  value = "${local_file.user-kubeconfig.filename}"
+  value = "${data.template_file.user-kubeconfig.rendered}"
+}
+
+# etcd TLS assets
+
+output "etcd_ca_cert" {
+  value = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+}
+
+output "etcd_client_cert" {
+  value = "${tls_locally_signed_cert.client.cert_pem}"
+}
+
+output "etcd_client_key" {
+  value = "${tls_private_key.client.private_key_pem}"
+}
+
+output "etcd_server_cert" {
+  value = "${tls_locally_signed_cert.server.cert_pem}"
+}
+
+output "etcd_server_key" {
+  value = "${tls_private_key.server.private_key_pem}"
+}
+
+output "etcd_peer_cert" {
+  value = "${tls_locally_signed_cert.peer.cert_pem}"
+}
+
+output "etcd_peer_key" {
+  value = "${tls_private_key.peer.private_key_pem}"
 }
 
 # Some platforms may need to reconstruct the kubeconfig directly in user-data.

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -8,20 +8,18 @@ spec:
   - name: kube-apiserver
     image: ${hyperkube_image}
     command:
-    - /usr/bin/flock
-    - --exclusive
-    - --timeout=30
-    - /var/lock/api-server.lock
     - /hyperkube
     - apiserver
-    - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --authorization-mode=RBAC
     - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+    - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
+    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
     - --etcd-servers=${etcd_servers}
-    - --insecure-port=0
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
     - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
     - --secure-port=443
@@ -29,7 +27,6 @@ spec:
     - --service-cluster-ip-range=${service_cidr}
     - --cloud-provider=${cloud_provider}
     - --storage-backend=etcd3
-    - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
     - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
     - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
     env:
@@ -54,7 +51,7 @@ spec:
       path: /etc/kubernetes/bootstrap-secrets
   - name: ssl-certs-host
     hostPath:
-      path: /usr/share/ca-certificates
+      path: ${trusted_certs_dir}
   - name: var-lock
     hostPath:
       path: /var/lock

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -12,6 +12,7 @@ spec:
     - controller-manager
     - --allocate-node-cidrs=true
     - --cluster-cidr=${pod_cidr}
+    - --service-cluster-ip-range=${service_cidr}
     - --cloud-provider=${cloud_provider}
     - --configure-cloud-routes=false
     - --kubeconfig=/etc/kubernetes/kubeconfig
@@ -32,4 +33,4 @@ spec:
       path: /etc/kubernetes
   - name: ssl-host
     hostPath:
-      path: /usr/share/ca-certificates
+      path: ${trusted_certs_dir}

resources/calico/bgpconfigurations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico BGP Configuration
+kind: CustomResourceDefinition
+metadata:
+  name: bgpconfigurations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPConfiguration
+    plural: bgpconfigurations
+    singular: bgpconfiguration

resources/calico/bgppeers-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico BGP Peers
+kind: CustomResourceDefinition
+metadata:
+  name: bgppeers.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPPeer
+    plural: bgppeers
+    singular: bgppeer

resources/calico/calico-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system

resources/calico/calico-cluster-role.yaml

@@ -0,0 +1,68 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: calico-node
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups: [""]
+    resources:
+      - services
+    verbs:
+      - get
+  - apiGroups: [""]
+    resources:
+      - endpoints
+    verbs:
+      - get
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - globalfelixconfigs
+      - felixconfigurations
+      - bgppeers
+      - globalbgpconfigs
+      - bgpconfigurations
+      - ippools
+      - globalnetworkpolicies
+      - globalnetworksets
+      - networkpolicies
+      - clusterinformations
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch

resources/calico/calico-config.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-config
+  namespace: kube-system
+data:
+  # Disable Typha for now.
+  typha_service_name: "none"
+  # The CNI network configuration to install on each node.
+  cni_network_config: |-
+    {
+      "name": "k8s-pod-network",
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "calico",
+          "log_level": "info",
+          "datastore_type": "kubernetes",
+          "nodename": "__KUBERNETES_NODE_NAME__",
+          "mtu": ${network_mtu},
+          "ipam": {
+            "type": "host-local",
+            "subnet": "usePodCidr"
+          },
+          "policy": {
+            "type": "k8s",
+            "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+          },
+          "kubernetes": {
+            "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+            "kubeconfig": "__KUBECONFIG_FILEPATH__"
+          }
+        },
+        {
+          "type": "portmap",
+          "snat": true,
+          "capabilities": {"portMappings": true}
+        }
+      ]
+    }

resources/calico/calico-service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system

resources/calico/calico.yaml

@@ -0,0 +1,152 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+    spec:
+      hostNetwork: true
+      serviceAccountName: calico-node
+      tolerations:
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
+      containers:
+        - name: calico-node
+          image: ${calico_image}
+          env:
+            # Use Kubernetes API as the backing datastore.
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+            # Enable felix info logging.
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "info"
+            # Cluster type to identify the deployment type
+            - name: CLUSTER_TYPE
+              value: "k8s,bgp"
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
+            # Disable IPV6 on Kubernetes.
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
+              value: "${network_mtu}"
+            # Wait for the datastore.
+            - name: WAIT_FOR_DATASTORE
+              value: "true"
+            # The Calico IPv4 pool CIDR (should match `--cluster-cidr`).
+            - name: CALICO_IPV4POOL_CIDR
+              value: "${pod_cidr}"
+            # Enable IPIP
+            - name: CALICO_IPV4POOL_IPIP
+              value: "Always"
+            # Enable IP-in-IP within Felix.
+            - name: FELIX_IPINIPENABLED
+              value: "true"
+            # Typha support: controlled by the ConfigMap.
+            - name: FELIX_TYPHAK8SSERVICENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: typha_service_name
+            # Set node name based on k8s nodeName.
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            # Auto-detect the BGP IP address.
+            - name: IP
+              value: "autodetect"
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 250m
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 9099
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 9099
+            periodSeconds: 10
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+            - mountPath: /var/lib/calico
+              name: var-lib-calico
+              readOnly: false
+        # Install Calico CNI binaries and CNI network config file on nodes
+        - name: install-cni
+          image: ${calico_cni_image}
+          command: ["/install-cni.sh"]
+          env:
+            # Name of the CNI config file to create on each node.
+            - name: CNI_CONF_NAME
+              value: "10-calico.conflist"
+            # Contents of the CNI config to create on each node.
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            # Set node name based on k8s nodeName
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CNI_NET_DIR
+              value: "/etc/kubernetes/cni/net.d"
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+      terminationGracePeriodSeconds: 0
+      volumes:
+        # Used by calico/node
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
+        # Used by install-cni
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/kubernetes/cni/net.d

resources/calico/clusterinformations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Cluster Information
+kind: CustomResourceDefinition
+metadata:
+  name: clusterinformations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: ClusterInformation
+    plural: clusterinformations
+    singular: clusterinformation

resources/calico/felixconfigurations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Felix Configuration
+kind: CustomResourceDefinition
+metadata:
+   name: felixconfigurations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: FelixConfiguration
+    plural: felixconfigurations
+    singular: felixconfiguration

resources/calico/globalnetworkpolicies-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworkpolicies.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkPolicy
+    plural: globalnetworkpolicies
+    singular: globalnetworkpolicy

resources/calico/globalnetworksets-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Sets
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworksets.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkSet
+    plural: globalnetworksets
+    singular: globalnetworkset

resources/calico/ippools-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico IP Pools
+kind: CustomResourceDefinition
+metadata:
+  name: ippools.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPPool
+    plural: ippools
+    singular: ippool

resources/calico/networkpolicies-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: networkpolicies.crd.projectcalico.org
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkPolicy
+    plural: networkpolicies
+    singular: networkpolicy

resources/etcd/bootstrap-etcd-service.json

@@ -1,26 +0,0 @@
-{
-  "apiVersion": "v1",
-  "kind": "Service",
-  "metadata": {
-    "name": "bootstrap-etcd-service",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "selector": {
-      "k8s-app": "boot-etcd"
-    },
-    "clusterIP": "${bootstrap_etcd_service_ip}",
-    "ports": [
-      {
-        "name": "client",
-        "port": 12379,
-        "protocol": "TCP"
-      },
-      {
-        "name": "peers",
-        "port": 12380,
-        "protocol": "TCP"
-      }
-    ]
-  }
-}

resources/etcd/migrate-etcd-cluster.json

@@ -1,27 +0,0 @@
-{
-  "apiVersion": "etcd.coreos.com/v1beta1",
-  "kind": "Cluster",
-  "metadata": {
-    "name": "kube-etcd",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "size": 1,
-    "version": "v3.1.6",
-    "pod": {
-      "nodeSelector": {
-        "node-role.kubernetes.io/master": ""
-      },
-      "tolerations": [
-        {
-          "key": "node-role.kubernetes.io/master",
-          "operator": "Exists",
-          "effect": "NoSchedule"
-        }
-      ]
-    },
-    "selfHosted": {
-      "bootMemberClientEndpoint": "http://${bootstrap_etcd_service_ip}:12379"
-    }
-  }
-}

resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml

@@ -1,29 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-etcd
-  namespace: kube-system
-  labels:
-    k8s-app: boot-etcd
-spec:
-  containers:
-  - name: etcd
-    image: ${etcd_image}
-    command:
-    - /usr/local/bin/etcd
-    - --name=boot-etcd
-    - --listen-client-urls=http://0.0.0.0:12379
-    - --listen-peer-urls=http://0.0.0.0:12380
-    - --advertise-client-urls=http://${bootstrap_etcd_service_ip}:12379
-    - --initial-advertise-peer-urls=http://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster=boot-etcd=http://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster-token=bootkube
-    - --initial-cluster-state=new
-    - --data-dir=/var/etcd/data
-    env:
-      - name: MY_POD_IP
-        valueFrom:
-          fieldRef:
-            fieldPath: status.podIP
-  hostNetwork: true
-  restartPolicy: Never

resources/experimental/manifests/etcd-operator.yaml

@@ -1,35 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
-  name: etcd-operator
-  namespace: kube-system
-  labels:
-    k8s-app: etcd-operator
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        k8s-app: etcd-operator
-    spec:
-      containers:
-      - name: etcd-operator
-        image: quay.io/coreos/etcd-operator:v0.3.0
-        command:
-          - /usr/local/bin/etcd-operator
-          - --analytics=false
-        env:
-        - name: MY_POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: MY_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule

resources/experimental/manifests/etcd-service.yaml

@@ -1,14 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-service
-  namespace: kube-system
-spec:
-  selector:
-    app: etcd
-    etcd_cluster: kube-etcd
-  clusterIP: ${etcd_service_ip}
-  ports:
-  - name: client
-    port: 2379
-    protocol: TCP

resources/flannel/flannel-cfg.yaml

@@ -0,0 +1,36 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    k8s-app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "cniVersion": "0.3.1",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "${pod_cidr}",
+      "Backend": {
+        "Type": "vxlan"
+      }
+    }

resources/flannel/flannel-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system

resources/flannel/flannel-cluster-role.yaml

@@ -0,0 +1,24 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch

resources/flannel/flannel-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system

resources/flannel/flannel.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-flannel
@@ -7,15 +7,20 @@ metadata:
     tier: node
     k8s-app: flannel
 spec:
+  selector:
+    matchLabels:
+      tier: node
+      k8s-app: flannel
   template:
     metadata:
       labels:
         tier: node
         k8s-app: flannel
     spec:
+      serviceAccountName: flannel
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.7.1-amd64
+        image: ${flannel_image}
         command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr", "--iface=$(POD_IP)"]
         securityContext:
           privileged: true
@@ -40,18 +45,25 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       - name: install-cni
-        image: busybox
-        command: [ "/bin/sh", "-c", "set -e -x; TMP=/etc/cni/net.d/.tmp-flannel-cfg; cp /etc/kube-flannel/cni-conf.json $TMP; mv $TMP /etc/cni/net.d/10-flannel.conf; while :; do sleep 3600; done" ]
+        image: ${flannel_cni_image}
+        command: ["/install-cni.sh"]
+        env:
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kube-flannel-cfg
+              key: cni-conf.json
         volumeMounts:
         - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
+          mountPath: /host/etc/cni/net.d
+        - name: host-cni-bin
+          mountPath: /host/opt/cni/bin/
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
         operator: Exists
-        effect: NoSchedule
       volumes:
         - name: run
           hostPath:
@@ -62,3 +74,10 @@ spec:
         - name: flannel-cfg
           configMap:
             name: kube-flannel-cfg
+        - name: host-cni-bin
+          hostPath:
+            path: /opt/cni/bin
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

resources/manifests/kube-apiserver-secret.yaml

@@ -9,3 +9,6 @@ data:
   apiserver.crt: ${apiserver_cert}
   service-account.pub: ${serviceaccount_pub}
   ca.crt: ${ca_cert}
+  etcd-client-ca.crt: ${etcd_ca_cert}
+  etcd-client.crt: ${etcd_client_cert}
+  etcd-client.key: ${etcd_client_key}

resources/manifests/kube-apiserver.yaml

@@ -1,4 +1,4 @@
-apiVersion: "extensions/v1beta1"
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-apiserver
@@ -7,6 +7,10 @@ metadata:
     tier: control-plane
     k8s-app: kube-apiserver
 spec:
+  selector:
+    matchLabels:
+      tier: control-plane
+      k8s-app: kube-apiserver
   template:
     metadata:
       labels:
@@ -14,19 +18,13 @@ spec:
         k8s-app: kube-apiserver
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-apiserver
         image: ${hyperkube_image}
         command:
-        - /usr/bin/flock
-        - --exclusive
-        - --timeout=30
-        - /var/lock/api-server.lock
         - /hyperkube
         - apiserver
-        - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota
         - --advertise-address=$(POD_IP)
         - --allow-privileged=true
         - --anonymous-auth=false
@@ -34,15 +32,17 @@ spec:
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider=${cloud_provider}
+        - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+        - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
+        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
         - --etcd-servers=${etcd_servers}
-        - --insecure-port=0
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
         - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
         - --secure-port=443
         - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
         - --service-cluster-ip-range=${service_cidr}
         - --storage-backend=etcd3
-        - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
         - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
         - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
         env:
@@ -64,18 +64,20 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
       volumes:
       - name: ssl-certs-host
         hostPath:
-          path: /usr/share/ca-certificates
+          path: ${trusted_certs_dir}
       - name: secrets
         secret:
           secretName: kube-apiserver
       - name: var-lock
         hostPath:
           path: /var/lock
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

resources/manifests/kube-controller-manager-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-controller-manager
+subjects:
+- kind: ServiceAccount
+  name: kube-controller-manager
+  namespace: kube-system

resources/manifests/kube-controller-manager-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: kube-controller-manager

resources/manifests/kube-controller-manager.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-controller-manager
@@ -8,13 +8,15 @@ metadata:
     k8s-app: kube-controller-manager
 spec:
   replicas: 2
+  selector:
+    matchLabels:
+      tier: control-plane
+      k8s-app: kube-controller-manager
   template:
     metadata:
       labels:
         tier: control-plane
         k8s-app: kube-controller-manager
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -30,7 +32,7 @@ spec:
                 - key: k8s-app
                   operator: In
                   values:
-                  - kube-contoller-manager
+                  - kube-controller-manager
               topologyKey: kubernetes.io/hostname
       containers:
       - name: kube-controller-manager
@@ -38,11 +40,14 @@ spec:
         command:
         - ./hyperkube
         - controller-manager
+        - --use-service-account-credentials
         - --allocate-node-cidrs=true
         - --cloud-provider=${cloud_provider}
         - --cluster-cidr=${pod_cidr}
+        - --service-cluster-ip-range=${service_cidr}
         - --configure-cloud-routes=false
         - --leader-elect=true
+        - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
         - --root-ca-file=/etc/kubernetes/secrets/ca.crt
         - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
         livenessProbe:
@@ -55,14 +60,19 @@ spec:
         - name: secrets
           mountPath: /etc/kubernetes/secrets
           readOnly: true
+        - name: volumeplugins
+          mountPath: /var/lib/kubelet/volumeplugins
+          readOnly: true          
         - name: ssl-host
           mountPath: /etc/ssl/certs
           readOnly: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
+      serviceAccountName: kube-controller-manager
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
@@ -72,5 +82,8 @@ spec:
           secretName: kube-controller-manager
       - name: ssl-host
         hostPath:
-          path: /usr/share/ca-certificates
+          path: ${trusted_certs_dir}
+      - name: volumeplugins
+        hostPath:
+          path: /var/lib/kubelet/volumeplugins          
       dnsPolicy: Default # Don't use cluster DNS.

resources/manifests/kube-dns-deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-dns
@@ -6,6 +6,7 @@ metadata:
   labels:
     k8s-app: kube-dns
     kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   # replicas: not specified here:
   # 1. In order to make Addon Manager do not reconcile this replicas parameter.
@@ -22,12 +23,21 @@ spec:
     metadata:
       labels:
         k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+      - name: kube-dns-config
+        configMap:
+          name: kube-dns
+          optional: true
       containers:
       - name: kubedns
-        image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1
+        image: ${kubedns_image}
         resources:
           # TODO: Set memory limits when we've profiled the container for large
           # clusters, then set request = limit to keep this container in
@@ -57,7 +67,7 @@ spec:
           initialDelaySeconds: 3
           timeoutSeconds: 5
         args:
-        - --domain=cluster.local.
+        - --domain=${cluster_domain_suffix}.
         - --dns-port=10053
         - --config-dir=/kube-dns-config
         - --v=2
@@ -78,7 +88,7 @@ spec:
         - name: kube-dns-config
           mountPath: /kube-dns-config
       - name: dnsmasq
-        image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1
+        image: ${kubedns_dnsmasq_image}
         livenessProbe:
           httpGet:
             path: /healthcheck/dnsmasq
@@ -96,8 +106,9 @@ spec:
         - --
         - -k
         - --cache-size=1000
+        - --no-negcache
         - --log-facility=-
-        - --server=/cluster.local/127.0.0.1#10053
+        - --server=/${cluster_domain_suffix}/127.0.0.1#10053
         - --server=/in-addr.arpa/127.0.0.1#10053
         - --server=/ip6.arpa/127.0.0.1#10053
         ports:
@@ -116,7 +127,7 @@ spec:
         - name: kube-dns-config
           mountPath: /etc/k8s/dns/dnsmasq-nanny
       - name: sidecar
-        image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1
+        image: ${kubedns_sidecar_image}
         livenessProbe:
           httpGet:
             path: /metrics
@@ -129,8 +140,8 @@ spec:
         args:
         - --v=2
         - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A
+        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cluster_domain_suffix},5,SRV
+        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cluster_domain_suffix},5,SRV
         ports:
         - containerPort: 10054
           name: metrics
@@ -140,16 +151,4 @@ spec:
             memory: 20Mi
             cpu: 10m
       dnsPolicy: Default  # Don't use cluster DNS.
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: kube-dns-config
-        configMap:
-          name: kube-dns
-          optional: true
+      serviceAccountName: kube-dns

resources/manifests/kube-dns-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-dns
+  namespace: kube-system

resources/manifests/kube-etcd-network-checkpointer.yaml

@@ -1,48 +0,0 @@
-apiVersion: "extensions/v1beta1"
-kind: DaemonSet
-metadata:
-  name: kube-etcd-network-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-etcd-network-checkpointer
-spec:
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-etcd-network-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-    spec:
-      containers:
-      - image: quay.io/coreos/kenc:48b6feceeee56c657ea9263f47b6ea091e8d3035
-        name: kube-etcd-network-checkpointer
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /etc/kubernetes/selfhosted-etcd
-          name: checkpoint-dir
-          readOnly: false
-        - mountPath: /var/lock
-          name: var-lock
-          readOnly: false
-        command:
-        - /usr/bin/flock
-        - /var/lock/kenc.lock
-        - -c
-        - "kenc -r -m iptables && kenc -m iptables"
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: checkpoint-dir
-        hostPath:
-          path: /etc/kubernetes/checkpoint-iptables
-      - name: var-lock
-        hostPath:
-          path: /var/lock

resources/manifests/kube-flannel-cfg.yaml

@@ -1,24 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: kube-flannel-cfg
-  namespace: kube-system
-  labels:
-    tier: node
-    k8s-app: flannel
-data:
-  cni-conf.json: |
-    {
-      "name": "cbr0",
-      "type": "flannel",
-      "delegate": {
-        "isDefaultGateway": true
-      }
-    }
-  net-conf.json: |
-    {
-      "Network": "${pod_cidr}",
-      "Backend": {
-        "Type": "vxlan"
-      }
-    }

resources/manifests/kube-proxy-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-proxy
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-proxier # Automatically created system role.
+subjects:
+- kind: ServiceAccount
+  name: kube-proxy
+  namespace: kube-system

resources/manifests/kube-proxy-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: kube-proxy

resources/manifests/kube-proxy.yaml

@@ -1,4 +1,4 @@
-apiVersion: "extensions/v1beta1"
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-proxy
@@ -7,19 +7,21 @@ metadata:
     tier: node
     k8s-app: kube-proxy
 spec:
+  selector:
+    matchLabels:
+      tier: node
+      k8s-app: kube-proxy
   template:
     metadata:
       labels:
         tier: node
         k8s-app: kube-proxy
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-proxy
         image: ${hyperkube_image}
         command:
-        - /hyperkube
+        - ./hyperkube
         - proxy
         - --cluster-cidr=${pod_cidr}
         - --hostname-override=$(NODE_NAME)
@@ -33,23 +35,33 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
         - mountPath: /etc/ssl/certs
           name: ssl-certs-host
           readOnly: true
-        - name: etc-kubernetes
+        - name: kubeconfig
           mountPath: /etc/kubernetes
           readOnly: true
       hostNetwork: true
+      serviceAccountName: kube-proxy
       tolerations:
-      - key: CriticalAddonsOnly
+      - effect: NoSchedule
         operator: Exists
-      - key: node-role.kubernetes.io/master
+      - effect: NoExecute
         operator: Exists
-        effect: NoSchedule
       volumes:
-      - hostPath:
-          path: /usr/share/ca-certificates
-        name: ssl-certs-host
-      - name: etc-kubernetes
+      - name: lib-modules
         hostPath:
-          path: /etc/kubernetes
+          path: /lib/modules
+      - name: ssl-certs-host
+        hostPath:
+          path: ${trusted_certs_dir}
+      - name: kubeconfig
+        configMap:
+          name: kubeconfig-in-cluster
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

resources/manifests/kube-scheduler.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-scheduler
@@ -8,13 +8,15 @@ metadata:
     k8s-app: kube-scheduler
 spec:
   replicas: 2
+  selector:
+    matchLabels:
+      tier: control-plane
+      k8s-app: kube-scheduler
   template:
     metadata:
       labels:
         tier: control-plane
         k8s-app: kube-scheduler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -47,9 +49,10 @@ spec:
           timeoutSeconds: 15
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kube-system-rbac-role-binding.yaml

@@ -1,4 +1,4 @@
-apiVersion: rbac.authorization.k8s.io/v1alpha1
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: system:default-sa

resources/manifests/kubeconfig-in-cluster.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubeconfig-in-cluster
+  namespace: kube-system
+data:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - name: local
+      cluster:
+        server: ${server}
+        certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    users:
+    - name: service-account
+      user:
+        # Use service account token
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    contexts:
+    - context:
+        cluster: local
+        user: service-account

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pod-checkpointer
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-checkpointer
+subjects:
+- kind: ServiceAccount
+  name: pod-checkpointer
+  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-checkpointer
+  namespace: kube-system
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["secrets", "configmaps"]
+  verbs: ["get"]

resources/manifests/pod-checkpointer-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: pod-checkpointer

resources/manifests/pod-checkpointer.yaml

@@ -1,4 +1,4 @@
-apiVersion: "extensions/v1beta1"
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: pod-checkpointer
@@ -7,6 +7,10 @@ metadata:
     tier: control-plane
     k8s-app: pod-checkpointer
 spec:
+  selector:
+    matchLabels:
+      tier: control-plane
+      k8s-app: pod-checkpointer
   template:
     metadata:
       labels:
@@ -16,12 +20,12 @@ spec:
         checkpointer.alpha.coreos.com/checkpoint: "true"
     spec:
       containers:
-      - name: checkpoint
-        image: quay.io/coreos/pod-checkpointer:2cad4cac4186611a79de1969e3ea4924f02f459e
+      - name: pod-checkpointer
+        image: ${pod_checkpointer_image}
         command:
         - /checkpoint
-        - --v=4
         - --lock-file=/var/run/lock/pod-checkpointer.lock
+        - --kubeconfig=/etc/checkpointer/kubeconfig
         env:
         - name: NODE_NAME
           valueFrom:
@@ -37,10 +41,13 @@ spec:
               fieldPath: metadata.namespace
         imagePullPolicy: Always
         volumeMounts:
+        - mountPath: /etc/checkpointer
+          name: kubeconfig
         - mountPath: /etc/kubernetes
           name: etc-kubernetes
         - mountPath: /var/run
           name: var-run
+      serviceAccountName: pod-checkpointer
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -50,9 +57,16 @@ spec:
         operator: Exists
         effect: NoSchedule
       volumes:
+      - name: kubeconfig
+        configMap:
+          name: kubeconfig-in-cluster
       - name: etc-kubernetes
         hostPath:
           path: /etc/kubernetes
       - name: var-run
         hostPath:
           path: /var/run
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

terraform.tfvars.example

@@ -1,5 +1,5 @@
 cluster_name = "example"
 api_servers = ["node1.example.com"]
-etcd_servers = ["http://127.0.0.1:2379"]
-asset_dir = "/home/core/clusters/mycluster"
-experimental_self_hosted_etcd = false
+etcd_servers = ["node1.example.com"]
+asset_dir = "/home/core/mycluster"
+networking = "flannel"

tls-etcd.tf

@@ -0,0 +1,199 @@
+# etcd-client-ca.crt
+resource "local_file" "etcd_client_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
+}
+
+# etcd-client.crt
+resource "local_file" "etcd_client_crt" {
+  content  = "${tls_locally_signed_cert.client.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd-client.crt"
+}
+
+# etcd-client.key
+resource "local_file" "etcd_client_key" {
+  content  = "${tls_private_key.client.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcd-client.key"
+}
+
+# server-ca.crt
+resource "local_file" "etcd_server_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
+}
+
+# server.crt
+resource "local_file" "etcd_server_crt" {
+  content  = "${tls_locally_signed_cert.server.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server.crt"
+}
+
+# server.key
+resource "local_file" "etcd_server_key" {
+  content  = "${tls_private_key.server.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server.key"
+}
+
+# peer-ca.crt
+resource "local_file" "etcd_peer_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
+}
+
+# peer.crt
+resource "local_file" "etcd_peer_crt" {
+  content  = "${tls_locally_signed_cert.peer.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/peer.crt"
+}
+
+# peer.key
+resource "local_file" "etcd_peer_key" {
+  content  = "${tls_private_key.peer.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcd/peer.key"
+}
+
+# certificates and keys
+
+resource "tls_private_key" "etcd-ca" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_self_signed_cert" "etcd-ca" {
+  key_algorithm   = "${tls_private_key.etcd-ca.algorithm}"
+  private_key_pem = "${tls_private_key.etcd-ca.private_key_pem}"
+
+  subject {
+    common_name  = "etcd-ca"
+    organization = "etcd"
+  }
+
+  is_ca_certificate     = true
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "cert_signing",
+  ]
+}
+
+# client certs are used for client (apiserver, locksmith, etcd-operator)
+# to etcd communication
+resource "tls_private_key" "client" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "client" {
+  key_algorithm   = "${tls_private_key.client.algorithm}"
+  private_key_pem = "${tls_private_key.client.private_key_pem}"
+
+  subject {
+    common_name  = "etcd-client"
+    organization = "etcd"
+  }
+
+  ip_addresses = [
+    "127.0.0.1",
+  ]
+
+  dns_names = ["${concat(
+    var.etcd_servers,
+    list(
+      "localhost",
+    ))}"]
+}
+
+resource "tls_locally_signed_cert" "client" {
+  cert_request_pem = "${tls_cert_request.client.cert_request_pem}"
+
+  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
+  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
+  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+    "client_auth",
+  ]
+}
+
+resource "tls_private_key" "server" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "server" {
+  key_algorithm   = "${tls_private_key.server.algorithm}"
+  private_key_pem = "${tls_private_key.server.private_key_pem}"
+
+  subject {
+    common_name  = "etcd-server"
+    organization = "etcd"
+  }
+
+  ip_addresses = [
+    "127.0.0.1",
+  ]
+
+  dns_names = ["${concat(
+    var.etcd_servers,
+    list(
+      "localhost",
+    ))}"]
+}
+
+resource "tls_locally_signed_cert" "server" {
+  cert_request_pem = "${tls_cert_request.server.cert_request_pem}"
+
+  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
+  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
+  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+    "client_auth",
+  ]
+}
+
+resource "tls_private_key" "peer" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "peer" {
+  key_algorithm   = "${tls_private_key.peer.algorithm}"
+  private_key_pem = "${tls_private_key.peer.private_key_pem}"
+
+  subject {
+    common_name  = "etcd-peer"
+    organization = "etcd"
+  }
+
+  dns_names = ["${var.etcd_servers}"]
+}
+
+resource "tls_locally_signed_cert" "peer" {
+  cert_request_pem = "${tls_cert_request.peer.cert_request_pem}"
+
+  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
+  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
+  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+    "client_auth",
+  ]
+}

tls-k8s.tf

@@ -70,7 +70,7 @@ resource "tls_cert_request" "apiserver" {
     "kubernetes",
     "kubernetes.default",
     "kubernetes.default.svc",
-    "kubernetes.default.svc.cluster.local",
+    "kubernetes.default.svc.${var.cluster_domain_suffix}",
   ]
 
   ip_addresses = [

variables.tf

@@ -4,20 +4,15 @@ variable "cluster_name" {
 }
 
 variable "api_servers" {
-  description = "URL used to reach kube-apiserver"
+  description = "List of URLs used to reach kube-apiserver"
   type        = "list"
 }
 
 variable "etcd_servers" {
-  description = "List of etcd server URLs including protocol, host, and port"
+  description = "List of URLs used to reach etcd servers."
   type        = "list"
 }
 
-variable "experimental_self_hosted_etcd" {
-  description = "(Experimental) Create self-hosted etcd assets"
-  default = false
-}
-
 variable "asset_dir" {
   description = "Path to a directory where generated assets should be placed (contains secrets)"
   type        = "string"
@@ -29,6 +24,18 @@ variable "cloud_provider" {
   default     = ""
 }
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
+variable "network_mtu" {
+  description = "CNI interface MTU (applies to calico only)"
+  type        = "string"
+  default     = "1500"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"
@@ -38,10 +45,17 @@ variable "pod_cidr" {
 variable "service_cidr" {
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
-The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 200th IP will be reserved for bootstrap self-hosted etcd.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
 EOD
+
+  type    = "string"
+  default = "10.3.0.0/24"
+}
+
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns"
   type        = "string"
-  default     = "10.3.0.0/24"
+  default     = "cluster.local"
 }
 
 variable "container_images" {
@@ -49,11 +63,24 @@ variable "container_images" {
   type        = "map"
 
   default = {
-    hyperkube = "quay.io/coreos/hyperkube:v1.6.2_coreos.0"
-    etcd = "quay.io/coreos/etcd:v3.1.6"
+    calico           = "quay.io/calico/node:v3.0.4"
+    calico_cni       = "quay.io/calico/cni:v2.0.1"
+    flannel          = "quay.io/coreos/flannel:v0.10.0-amd64"
+    flannel_cni      = "quay.io/coreos/flannel-cni:v0.3.0"
+    hyperkube        = "k8s.gcr.io/hyperkube:v1.10.0"
+    kubedns          = "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.9"
+    kubedns_dnsmasq  = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.9"
+    kubedns_sidecar  = "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.9"
+    pod_checkpointer = "quay.io/coreos/pod-checkpointer:9dc83e1ab3bc36ca25c9f7c18ddef1b91d4a0558"
   }
 }
 
+variable "trusted_certs_dir" {
+  description = "Path to the directory on cluster nodes where trust TLS certs are kept"
+  type        = "string"
+  default     = "/usr/share/ca-certificates"
+}
+
 variable "ca_certificate" {
   description = "Existing PEM-encoded CA certificate (generated if blank)"
   type        = "string"