*: Upgrade manifests for Kubernetes v1.6.6 and bootkube v0.4.5

* Enable TLS for experimental self-hosted etcd * Update the flannel Daemonset based on upstream * Switch control plane components to run as non-root * Add UpdateStrategy to control plane components
2026-01-27 10:20:45 +00:00 · 2017-06-21 11:37:11 -07:00
parent dd26460395
commit 99f50c5317
20 changed files with 264 additions and 126 deletions
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Use the `bootkube-terraform` module within your existing Terraform configs. Prov

 ```hcl
 module "bootkube" {
-  source = "git://https://github.com/dghubble/bootkube-terraform.git"
+  source = "git://https://github.com/dghubble/bootkube-terraform.git?ref=SHA"

  cluster_name = "example"
  api_servers = ["node1.example.com"]
@@ -30,7 +30,7 @@ terraform apply

 ### Comparison

-Render bootkube assets directly with bootkube v0.4.4.
+Render bootkube assets directly with bootkube v0.4.5.

 #### On-host etcd

@@ -41,7 +41,7 @@ bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 -
 Compare assets. The only diffs you should see are TLS credentials.

 ```sh
-diff -rw assets /home/core/cluster/mycluster
+diff -rw assets /home/core/mycluster
 ```

 #### Self-hosted etcd
@@ -53,6 +53,10 @@ bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 -
 Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment.

 ```sh
-diff -rw assets /home/core/cluster/mycluster
+pushd /home/core/mycluster
+mv experimental/bootstrap-manifests/* boostrap-manifests
+mv experimental/manifests/* manifests
+popd
+diff -rw assets /home/core/mycluster
 ```

--- a/assets-etcd.tf
+++ b/assets-etcd.tf
@@ -1,72 +1,43 @@
-# Experimental self-hosted etcd
+# Assets generated only when experimental self-hosted etcd is enabled

-# etcd pod and service bootstrap-manifests
-
-data "template_file" "bootstrap-etcd" {
-  template = "${file("${path.module}/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml")}"
+# bootstrap-etcd.yaml pod bootstrap-manifest
+resource "template_dir" "experimental-bootstrap-manifests" {
+  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
+  destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"

  vars {
    etcd_image                = "${var.container_images["etcd"]}"
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
+    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
  }
 }

-resource "local_file" "bootstrap-etcd" {
-  count    = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.bootstrap-etcd.rendered}"
-  filename = "${var.asset_dir}/experimental/bootstrap-manifests/bootstrap-etcd.yaml"
-}
-
-data "template_file" "bootstrap-etcd-service" {
-  template = "${file("${path.module}/resources/etcd/bootstrap-etcd-service.json")}"
-
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
-  }
-}
-
-resource "local_file" "bootstrap-etcd-service" {
-  count    = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.bootstrap-etcd-service.rendered}"
-  filename = "${var.asset_dir}/etcd/bootstrap-etcd-service.json"
-}
-
-data "template_file" "etcd-tpr" {
-  template = "${file("${path.module}/resources/etcd/migrate-etcd-cluster.json")}"
-
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 200)}"
-  }
-}
-
-resource "local_file" "etcd-tpr" {
-  count    = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  content  = "${data.template_file.etcd-tpr.rendered}"
-  filename = "${var.asset_dir}/etcd/migrate-etcd-cluster.json"
-}
-
-# etcd operator deployment and service manifests
-
-resource "local_file" "etcd-operator" {
+# etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
+resource "template_dir" "etcd-subfolder" {
  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  depends_on = ["template_dir.manifests"]
+  source_dir      = "${path.module}/resources/etcd"
+  destination_dir = "${var.asset_dir}/etcd"

-  content  = "${file("${path.module}/resources/experimental/manifests/etcd-operator.yaml")}"
-  filename = "${var.asset_dir}/experimental/manifests/etcd-operator.yaml"
+  vars {
+    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
+  }
 }

-data "template_file" "etcd-service" {
-  template = "${file("${path.module}/resources/experimental/manifests/etcd-service.yaml")}"
+# etcd-operator deployment and etcd-service manifests
+# etcd member peer, member client, and operator client secrets
+resource "template_dir" "experimental-manifests" {
+  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  source_dir      = "${path.module}/resources/experimental/manifests"
+  destination_dir = "${var.asset_dir}/experimental/manifests"

  vars {
    etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
+
+    # Self-hosted etcd TLS certs / keys
+    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_peer_cert = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
+    etcd_peer_key = "${base64encode(tls_private_key.peer.private_key_pem)}"
+    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
+    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" 
  }
 }
-
-resource "local_file" "etcd-service" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  depends_on = ["template_dir.manifests"]
-
-  content  = "${data.template_file.etcd-service.rendered}"
-  filename = "${var.asset_dir}/experimental/manifests/etcd-service.yaml"
-}
--- a/assets.tf
+++ b/assets.tf
@@ -1,20 +1,15 @@
-# Self-hosted Kubernetes bootstrap manifests
+# Self-hosted Kubernetes bootstrap-manifests
 resource "template_dir" "bootstrap-manifests" {
  source_dir      = "${path.module}/resources/bootstrap-manifests"
  destination_dir = "${var.asset_dir}/bootstrap-manifests"

  vars {
    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
+    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379,https://127.0.0.1:12379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"

    cloud_provider = "${var.cloud_provider}"
    pod_cidr       = "${var.pod_cidr}"
    service_cidr   = "${var.service_cidr}"
-
-    # TODO: Enable TLS for self-hosted etcd and remove these variables
-    etcd_ca_flag          = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt"}"
-    etcd_client_cert_flag = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt"}"
-    etcd_client_key_flag  = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key"}"
  }
 }

@@ -25,10 +20,9 @@ resource "template_dir" "manifests" {

  vars {
    hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("http://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
+    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"

    cloud_provider = "${var.cloud_provider}"
-
    pod_cidr            = "${var.pod_cidr}"
    service_cidr        = "${var.service_cidr}"
    kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
@@ -42,15 +36,21 @@ resource "template_dir" "manifests" {
    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}"
-
-    # TODO: Enable TLS for self-hosted etcd and remove these variables
-    etcd_ca_flag          = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt"}"
-    etcd_client_cert_flag = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt"}"
-    etcd_client_key_flag  = "${var.experimental_self_hosted_etcd ? "# etcd TLS not supported" : "- --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key"}"
  }
 }

-# Generated kubeconfig (auth/kubeconfig)
+# Generated kubeconfig
+resource "local_file" "kubeconfig" {
+  content  = "${data.template_file.kubeconfig.rendered}"
+  filename = "${var.asset_dir}/auth/kubeconfig"
+}
+
+# Generated kubeconfig with user-context
+resource "local_file" "user-kubeconfig" {
+  content  = "${data.template_file.user-kubeconfig.rendered}"
+  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
+}
+
 data "template_file" "kubeconfig" {
  template = "${file("${path.module}/resources/kubeconfig")}"

@@ -62,12 +62,6 @@ data "template_file" "kubeconfig" {
  }
 }

-resource "local_file" "kubeconfig" {
-  content  = "${data.template_file.kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/kubeconfig"
-}
-
-# Generated kubeconfig (auth/kubeconfig)
 data "template_file" "user-kubeconfig" {
  template = "${file("${path.module}/resources/user-kubeconfig")}"

@@ -80,7 +74,3 @@ data "template_file" "user-kubeconfig" {
  }
 }

-resource "local_file" "user-kubeconfig" {
-  content  = "${data.template_file.user-kubeconfig.rendered}"
-  filename = "${var.asset_dir}/auth/${var.cluster_name}-config"
-}
--- a/resources/bootstrap-manifests/bootstrap-apiserver.yaml
+++ b/resources/bootstrap-manifests/bootstrap-apiserver.yaml
@@ -9,8 +9,6 @@ spec:
    image: ${hyperkube_image}
    command:
    - /usr/bin/flock
-    - --exclusive
-    - --timeout=30
    - /var/lock/api-server.lock
    - /hyperkube
    - apiserver
@@ -20,10 +18,10 @@ spec:
    - --authorization-mode=RBAC
    - --bind-address=0.0.0.0
    - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt 
+    - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+    - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
    - --etcd-servers=${etcd_servers}
-    ${etcd_ca_flag}
-    ${etcd_client_cert_flag}
-    ${etcd_client_key_flag}
    - --insecure-port=0
    - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
    - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
--- a/resources/etcd/migrate-etcd-cluster.json
+++ b/resources/etcd/migrate-etcd-cluster.json
@@ -7,7 +7,7 @@
  },
  "spec": {
    "size": 1,
-    "version": "v3.1.6",
+    "version": "v3.1.8",
    "pod": {
      "nodeSelector": {
        "node-role.kubernetes.io/master": ""
@@ -21,7 +21,16 @@
      ]
    },
    "selfHosted": {
-      "bootMemberClientEndpoint": "http://${bootstrap_etcd_service_ip}:12379"
+      "bootMemberClientEndpoint": "https://${bootstrap_etcd_service_ip}:12379"
+    },
+    "TLS": {
+      "static": {
+        "member": {
+          "peerSecret": "etcd-member-peer-tls",
+          "clientSecret": "etcd-member-client-tls"
+        },
+        "operatorSecret": "etcd-operator-client-tls"
+      }
    }
  }
 }
--- a/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml
+++ b/resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml
@@ -12,18 +12,30 @@ spec:
    command:
    - /usr/local/bin/etcd
    - --name=boot-etcd
-    - --listen-client-urls=http://0.0.0.0:12379
-    - --listen-peer-urls=http://0.0.0.0:12380
-    - --advertise-client-urls=http://${bootstrap_etcd_service_ip}:12379
-    - --initial-advertise-peer-urls=http://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster=boot-etcd=http://${bootstrap_etcd_service_ip}:12380
+    - --listen-client-urls=https://0.0.0.0:12379
+    - --listen-peer-urls=https://0.0.0.0:12380
+    - --advertise-client-urls=https://${bootstrap_etcd_service_ip}:12379
+    - --initial-advertise-peer-urls=https://${bootstrap_etcd_service_ip}:12380
+    - --initial-cluster=boot-etcd=https://${bootstrap_etcd_service_ip}:12380
    - --initial-cluster-token=bootkube
    - --initial-cluster-state=new
    - --data-dir=/var/etcd/data
-    env:
-      - name: MY_POD_IP
-        valueFrom:
-          fieldRef:
-            fieldPath: status.podIP
+    - --peer-client-cert-auth=true
+    - --peer-trusted-ca-file=/etc/kubernetes/secrets/etcdMember/peer-ca-crt.pem
+    - --peer-cert-file=/etc/kubernetes/secrets/etcdMember/peer-crt.pem
+    - --peer-key-file=/etc/kubernetes/secrets/etcdMember/peer-key.pem
+    - --client-cert-auth=true
+    - --trusted-ca-file=/etc/kubernetes/secrets/etcdMember/client-ca-crt.pem
+    - --cert-file=/etc/kubernetes/secrets/etcdMember/client-crt.pem
+    - --key-file=/etc/kubernetes/secrets/etcdMember/client-key.pem
+    volumeMounts:
+      - mountPath: /etc/kubernetes/secrets
+        name: secrets
+        readOnly: true
+  volumes:
+  - name: secrets
+    hostPath:
+      path: /etc/kubernetes/bootstrap-secrets 
  hostNetwork: true
  restartPolicy: Never
+  dnsPolicy: ClusterFirstWithHostNet 
--- a/resources/experimental/manifests/etcd-member-client-tls.yaml
+++ b/resources/experimental/manifests/etcd-member-client-tls.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-member-client-tls
+  namespace: kube-system
+type: Opaque
+data:
+  client-ca-crt.pem: ${etcd_ca_cert}
+  client-crt.pem: ${etcd_client_cert}
+  client-key.pem: ${etcd_client_key}
--- a/resources/experimental/manifests/etcd-member-peer-tls.yaml
+++ b/resources/experimental/manifests/etcd-member-peer-tls.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-member-peer-tls
+  namespace: kube-system
+type: Opaque
+data:
+  peer-ca-crt.pem: ${etcd_ca_cert}
+  peer-crt.pem: ${etcd_peer_cert}
+  peer-key.pem: ${etcd_peer_key}
--- a/resources/experimental/manifests/etcd-operator-client-tls.yaml
+++ b/resources/experimental/manifests/etcd-operator-client-tls.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-operator-client-tls
+  namespace: kube-system
+type: Opaque
+data:
+  etcd-ca-crt.pem: ${etcd_ca_cert}
+  etcd-crt.pem: ${etcd_client_cert}
+  etcd-key.pem: ${etcd_client_key}
--- a/resources/experimental/manifests/etcd-operator.yaml
+++ b/resources/experimental/manifests/etcd-operator.yaml
@@ -6,6 +6,11 @@ metadata:
  labels:
    k8s-app: etcd-operator
 spec:
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 1
  replicas: 1
  template:
    metadata:
@@ -14,7 +19,7 @@ spec:
    spec:
      containers:
      - name: etcd-operator
-        image: quay.io/coreos/etcd-operator:v0.3.0
+        image: quay.io/coreos/etcd-operator:v0.3.3
        command:
          - /usr/local/bin/etcd-operator
          - --analytics=false
@@ -29,6 +34,9 @@ spec:
              fieldPath: metadata.name
      nodeSelector:
        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
      tolerations:
      - key: node-role.kubernetes.io/master
        operator: Exists
--- a/resources/manifests/kube-apiserver.yaml
+++ b/resources/manifests/kube-apiserver.yaml
@@ -21,8 +21,6 @@ spec:
        image: ${hyperkube_image}
        command:
        - /usr/bin/flock
-        - --exclusive
-        - --timeout=30
        - /var/lock/api-server.lock
        - /hyperkube
        - apiserver
@@ -33,10 +31,10 @@ spec:
        - --authorization-mode=RBAC
        - --bind-address=0.0.0.0
        - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-        ${etcd_ca_flag}
-        ${etcd_client_cert_flag}
-        ${etcd_client_key_flag}
        - --cloud-provider=${cloud_provider}
+        - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt 
+        - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
+        - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
        - --etcd-servers=${etcd_servers}
        - --insecure-port=0
        - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
@@ -82,3 +80,7 @@ spec:
      - name: var-lock
        hostPath:
          path: /var/lock
+  updateStrategy:
+      rollingUpdate:
+        maxUnavailable: 1
+      type: RollingUpdate
--- a/resources/manifests/kube-controller-manager.yaml
+++ b/resources/manifests/kube-controller-manager.yaml
@@ -60,6 +60,9 @@ spec:
          readOnly: true
      nodeSelector:
        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
--- a/resources/manifests/kube-etcd-network-checkpointer.yaml
+++ b/resources/manifests/kube-etcd-network-checkpointer.yaml
@@ -16,7 +16,7 @@ spec:
        checkpointer.alpha.coreos.com/checkpoint: "true"
    spec:
      containers:
-      - image: quay.io/coreos/kenc:48b6feceeee56c657ea9263f47b6ea091e8d3035
+      - image: quay.io/coreos/kenc:8f6e2e885f790030fbbb0496ea2a2d8830e58b8f
        name: kube-etcd-network-checkpointer
        securityContext:
          privileged: true
@@ -24,6 +24,9 @@ spec:
        - mountPath: /etc/kubernetes/selfhosted-etcd
          name: checkpoint-dir
          readOnly: false
+        - mountPath: /var/etcd
+          name: etcd-dir
+          readOnly: false
        - mountPath: /var/lock
          name: var-lock
          readOnly: false
@@ -43,6 +46,13 @@ spec:
      - name: checkpoint-dir
        hostPath:
          path: /etc/kubernetes/checkpoint-iptables
+      - name: etcd-dir
+        hostPath:
+          path: /var/etcd
      - name: var-lock
        hostPath:
          path: /var/lock
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
--- a/resources/manifests/kube-flannel.yaml
+++ b/resources/manifests/kube-flannel.yaml
@@ -40,13 +40,19 @@ spec:
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      - name: install-cni
-        image: busybox
-        command: [ "/bin/sh", "-c", "set -e -x; TMP=/etc/cni/net.d/.tmp-flannel-cfg; cp /etc/kube-flannel/cni-conf.json $TMP; mv $TMP /etc/cni/net.d/10-flannel.conf; while :; do sleep 3600; done" ]
+        image: quay.io/coreos/flannel-cni:0.1.0
+        command: ["/install-cni.sh"]
+        env:
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kube-flannel-cfg
+              key: cni-conf.json
        volumeMounts:
        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
+          mountPath: /host/etc/cni/net.d
+        - name: host-cni-bin
+          mountPath: /host/opt/cni/bin/
      hostNetwork: true
      tolerations:
      - key: node-role.kubernetes.io/master
@@ -62,3 +68,10 @@ spec:
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
+        - name: host-cni-bin
+          hostPath:
+            path: /opt/cni/bin
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate
--- a/resources/manifests/kube-proxy.yaml
+++ b/resources/manifests/kube-proxy.yaml
@@ -19,7 +19,7 @@ spec:
      - name: kube-proxy
        image: ${hyperkube_image}
        command:
-        - /hyperkube
+        - ./hyperkube
        - proxy
        - --cluster-cidr=${pod_cidr}
        - --hostname-override=$(NODE_NAME)
@@ -53,3 +53,7 @@ spec:
      - name: etc-kubernetes
        hostPath:
          path: /etc/kubernetes
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate 
--- a/resources/manifests/kube-scheduler.yaml
+++ b/resources/manifests/kube-scheduler.yaml
@@ -47,6 +47,9 @@ spec:
          timeoutSeconds: 15
      nodeSelector:
        node-role.kubernetes.io/master: ""
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 65534
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
--- a/resources/manifests/pod-checkpointer.yaml
+++ b/resources/manifests/pod-checkpointer.yaml
@@ -16,8 +16,8 @@ spec:
        checkpointer.alpha.coreos.com/checkpoint: "true"
    spec:
      containers:
-      - name: checkpoint
-        image: quay.io/coreos/pod-checkpointer:2cad4cac4186611a79de1969e3ea4924f02f459e
+      - name: pod-checkpointer
+        image: quay.io/coreos/pod-checkpointer:4e7a7dab10bc4d895b66c21656291c6e0b017248
        command:
        - /checkpoint
        - --v=4
@@ -56,3 +56,7 @@ spec:
      - name: var-run
        hostPath:
          path: /var/run
+  updateStrategy:
+      rollingUpdate:
+        maxUnavailable: 1
+      type: RollingUpdate
--- a/terraform.tfvars.example
+++ b/terraform.tfvars.example
@@ -1,5 +1,5 @@
 cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
-asset_dir = "/home/core/clusters/mycluster"
+asset_dir = "/home/core/mycluster"
 experimental_self_hosted_etcd = false
--- a/tls-etcd.tf
+++ b/tls-etcd.tf
@@ -28,6 +28,62 @@ resource "local_file" "etcd_peer_key" {
  filename = "${var.asset_dir}/tls/etcd-peer.key"
 }

+# add certs / keys for self-hosted etcd
+
+# operator/etcd-ca-crt.pem
+resource "local_file" "etcd_operator_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/operator/etcd-ca-crt.pem"
+}
+
+# operator/etcd-crt.pem
+resource "local_file" "etcd_operator_client_crt" {
+  content  = "${tls_locally_signed_cert.client.cert_pem}"
+  filename = "${var.asset_dir}/tls/operator/etcd-crt.pem"
+}
+
+# operator/etcd-key.pem
+resource "local_file" "etcd_operator_client_key" {
+  content  = "${tls_private_key.client.private_key_pem}"
+  filename = "${var.asset_dir}/tls/operator/etcd-key.pem"
+}
+
+# etcdMember/client-ca-crt.pem
+resource "local_file" "etcd_member_client_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/client-ca-crt.pem"
+}
+
+# etcdMember/client-crt.pem
+resource "local_file" "etcd_member_client_crt" {
+  content  = "${tls_locally_signed_cert.client.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/client-crt.pem"
+}
+
+# etcdMember/client-key.pem
+resource "local_file" "etcd_member_client_key" {
+  content  = "${tls_private_key.client.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/client-key.pem"
+}
+
+# etcdMember/peer-ca-crt.pem
+resource "local_file" "etcd_member_peer_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/peer-ca-crt.pem"
+}
+
+# etcdMember/peer-crt.pem
+resource "local_file" "etcd_member_peer_crt" {
+  content  = "${tls_locally_signed_cert.peer.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/peer-crt.pem"
+}
+
+# etcdMember/peer-key.pem
+resource "local_file" "etcd_member_peer_key" {
+  content  = "${tls_private_key.peer.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcdMember/peer-key.pem"
+}
+
 # certificates and keys

 resource "tls_private_key" "etcd-ca" {
@@ -64,11 +120,23 @@ resource "tls_cert_request" "client" {
  private_key_pem = "${tls_private_key.client.private_key_pem}"

  subject {
-    common_name  = "etcd"
+    common_name  = "etcd-client"
    organization = "etcd"
  }

-  dns_names = ["${var.etcd_servers}"]
+  ip_addresses = [
+    "127.0.0.1",
+    "${cidrhost(var.service_cidr, 15)}",
+    "${cidrhost(var.service_cidr, 20)}",
+  ]
+
+  dns_names = "${concat(
+    var.etcd_servers,
+    list(
+      "localhost",
+      "*.kube-etcd.kube-system.svc.cluster.local",
+      "kube-etcd-client.kube-system.svc.cluster.local",
+    ))}"
 }

 resource "tls_locally_signed_cert" "client" {
@@ -98,11 +166,20 @@ resource "tls_cert_request" "peer" {
  private_key_pem = "${tls_private_key.peer.private_key_pem}"

  subject {
-    common_name  = "etcd"
+    common_name  = "etcd-peer"
    organization = "etcd"
  }
+  
+  dns_names = "${concat(
+    var.etcd_servers,
+    list(
+      "*.kube-etcd.kube-system.svc.cluster.local",
+      "kube-etcd-client.kube-system.svc.cluster.local",
+    ))}"

-  dns_names = ["${var.etcd_servers}"]
+  ip_addresses = [
+    "${cidrhost(var.service_cidr, 20)}"
+  ]
 }

 resource "tls_locally_signed_cert" "peer" {
--- a/variables.tf
+++ b/variables.tf
@@ -9,7 +9,7 @@ variable "api_servers" {
 }

 variable "etcd_servers" {
-  description = "List of etcd server URLs including protocol, host, and port"
+  description = "List of etcd server URLs including protocol, host, and port. Ignored if experimental self-hosted etcd is enabled."
  type        = "list"
 }

@@ -38,7 +38,7 @@ variable "pod_cidr" {
 variable "service_cidr" {
  description = <<EOD
 CIDR IP range to assign Kubernetes services.
-The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 200th IP will be reserved for bootstrap self-hosted etcd.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 20th IP will be reserved for bootstrap self-hosted etcd.
 EOD

  type    = "string"
@@ -50,8 +50,8 @@ variable "container_images" {
  type        = "map"

  default = {
-    hyperkube = "quay.io/coreos/hyperkube:v1.6.4_coreos.0"
-    etcd      = "quay.io/coreos/etcd:v3.1.6"
+    hyperkube = "quay.io/coreos/hyperkube:v1.6.6_coreos.1"
+    etcd      = "quay.io/coreos/etcd:v3.1.8"
  }
 }