Update assets generation for bootkube v0.6.2

* Update hyperkube to v1.7.5_coreos.0
* Update etcd-operator to v0.5.0
* Update pod-checkpointer
* Update flannel-cni to v0.2.0
* Change etcd-operator TPR to CRD

.gitignore

@@ -1,2 +1,4 @@
 *.tfvars
+.terraform
 *.tfstate*
+assets

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Dalton Hubble
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -1,10 +1,14 @@
 # bootkube-terraform
 
-`bootkube-terraform` is a Terraform module that renders [bootkube](https://github.com/kubernetes-incubator/bootkube) assets, just like running the binary `bootkube render`. It aims to provide the same variable names, defaults, features, and outputs.
+`bootkube-terraform` is Terraform module that renders [kubernetes-incubator/bootkube](https://github.com/kubernetes-incubator/bootkube) bootstrapping assets. It functions as a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution.
+
+The module provides many of the same variable names, defaults, features, and outputs as running `bootkube render` directly.
 
 ## Usage
 
-Use the `bootkube-terraform` module within your existing Terraform configs. Provide the variables listed in `variables.tf` or check `terraform.tfvars.example` for examples.
+Use [Typhoon](https://github.com/poseidon/typhoon) to create and manage Kubernetes clusters in different environments. Use `bootkube-terraform` if you require low-level customizations to the control plane or wish to build your own distribution.
+
+Add the `bootkube-terraform` module alongside existing Terraform configs. Provide the variables listed in `variables.tf` or check `terraform.tfvars.example` for examples.
 
 ```hcl
 module "bootkube" {
@@ -30,7 +34,7 @@ terraform apply
 
 ### Comparison
 
-Render bootkube assets directly with bootkube v0.4.5.
+Render bootkube assets directly with bootkube v0.6.2.
 
 #### On-host etcd
 
@@ -41,6 +45,9 @@ bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 -
 Compare assets. The only diffs you should see are TLS credentials.
 
 ```sh
+pushd /home/core/mycluster
+mv manifests-networking/* manifests
+popd
 diff -rw assets /home/core/mycluster
 ```
 
@@ -56,6 +63,7 @@ Compare assets. Note that experimental must be generated to a separate directory
 pushd /home/core/mycluster
 mv experimental/bootstrap-manifests/* boostrap-manifests
 mv experimental/manifests/* manifests
+mv manifests-networking/* manifests
 popd
 diff -rw assets /home/core/mycluster
 ```

assets-etcd.tf

@@ -1,43 +0,0 @@
-# Assets generated only when experimental self-hosted etcd is enabled
-
-# bootstrap-etcd.yaml pod bootstrap-manifest
-resource "template_dir" "experimental-bootstrap-manifests" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"
-
-  vars {
-    etcd_image                = "${var.container_images["etcd"]}"
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
-resource "template_dir" "etcd-subfolder" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/etcd"
-  destination_dir = "${var.asset_dir}/etcd"
-
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd-operator deployment and etcd-service manifests
-# etcd member peer, member client, and operator client secrets
-resource "template_dir" "experimental-manifests" {
-  count      = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/manifests"
-  destination_dir = "${var.asset_dir}/experimental/manifests"
-
-  vars {
-    etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
-
-    # Self-hosted etcd TLS certs / keys
-    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_peer_cert = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
-    etcd_peer_key = "${base64encode(tls_private_key.peer.private_key_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}" 
-  }
-}

assets.tf

@@ -22,7 +22,7 @@ resource "template_dir" "manifests" {
     hyperkube_image = "${var.container_images["hyperkube"]}"
     etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
 
-    cloud_provider = "${var.cloud_provider}"
+    cloud_provider      = "${var.cloud_provider}"
     pod_cidr            = "${var.pod_cidr}"
     service_cidr        = "${var.service_cidr}"
     kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
@@ -33,9 +33,9 @@ resource "template_dir" "manifests" {
     serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"
     serviceaccount_key = "${base64encode(tls_private_key.service-account.private_key_pem)}"
 
-    etcd_ca_cert = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
     etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key = "${base64encode(tls_private_key.client.private_key_pem)}"
+    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
   }
 }
 
@@ -73,4 +73,3 @@ data "template_file" "user-kubeconfig" {
     server       = "${format("https://%s:443", element(var.api_servers, 0))}"
   }
 }
-

conditional.tf

@@ -0,0 +1,66 @@
+# Assets generated only when experimental self-hosted etcd is enabled
+
+resource "template_dir" "flannel-manifests" {
+  count           = "${var.networking == "flannel" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/flannel"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    pod_cidr = "${var.pod_cidr}"
+  }
+}
+
+resource "template_dir" "calico-manifests" {
+  count           = "${var.networking == "calico" ? 1 : 0}"
+  source_dir      = "${path.module}/resources/calico"
+  destination_dir = "${var.asset_dir}/manifests-networking"
+
+  vars {
+    network_mtu = "${var.network_mtu}"
+    pod_cidr    = "${var.pod_cidr}"
+  }
+}
+
+# bootstrap-etcd.yaml pod bootstrap-manifest
+resource "template_dir" "experimental-bootstrap-manifests" {
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
+  destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"
+
+  vars {
+    etcd_image                = "${var.container_images["etcd"]}"
+    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
+  }
+}
+
+# etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
+resource "template_dir" "etcd-subfolder" {
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  source_dir      = "${path.module}/resources/etcd"
+  destination_dir = "${var.asset_dir}/etcd"
+
+  vars {
+    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
+  }
+}
+
+# etcd-operator deployment and etcd-service manifests
+# etcd client, server, and peer tls secrets
+resource "template_dir" "experimental-manifests" {
+  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
+  source_dir      = "${path.module}/resources/experimental/manifests"
+  destination_dir = "${var.asset_dir}/experimental/manifests"
+
+  vars {
+    etcd_service_ip = "${cidrhost(var.service_cidr, 15)}"
+
+    # Self-hosted etcd TLS certs / keys
+    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
+    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
+    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
+    etcd_server_cert = "${base64encode(tls_locally_signed_cert.server.cert_pem)}"
+    etcd_server_key  = "${base64encode(tls_private_key.server.private_key_pem)}"
+    etcd_peer_cert   = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
+    etcd_peer_key    = "${base64encode(tls_private_key.peer.private_key_pem)}"
+  }
+}

outputs.tf

@@ -36,6 +36,14 @@ output "etcd_client_key" {
   value = "${tls_private_key.client.private_key_pem}"
 }
 
+output "etcd_server_cert" {
+  value = "${tls_locally_signed_cert.server.cert_pem}"
+}
+
+output "etcd_server_key" {
+  value = "${tls_private_key.server.private_key_pem}"
+}
+
 output "etcd_peer_cert" {
   value = "${tls_locally_signed_cert.peer.cert_pem}"
 }

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -18,9 +18,10 @@ spec:
     - --authorization-mode=RBAC
     - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
-    - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt 
+    - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
     - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
     - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
+    - --etcd-quorum-read=true
     - --etcd-servers=${etcd_servers}
     - --insecure-port=0
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt

resources/calico/calico-bgp-peers.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico BGP Peers
+kind: CustomResourceDefinition
+metadata:
+  name: bgppeers.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPPeer
+    plural: bgppeers
+    singular: bgppeer

resources/calico/calico-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: calico-node
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: calico-node
+subjects:
+- kind: ServiceAccount
+  name: calico-node
+  namespace: kube-system

resources/calico/calico-cluster-role.yaml

@@ -0,0 +1,53 @@
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: calico-node
+  namespace: kube-system
+rules:
+  - apiGroups: [""]
+    resources:
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - pods/status
+    verbs:
+      - update
+  - apiGroups: [""]
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: [""]
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups: ["extensions"]
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups: ["crd.projectcalico.org"]
+    resources:
+      - globalfelixconfigs
+      - bgppeers
+      - globalbgpconfigs
+      - ippools
+      - globalnetworkpolicies
+    verbs:
+      - create
+      - get
+      - list
+      - update
+      - watch

resources/calico/calico-config.yaml

@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-config
+  namespace: kube-system
+data:
+  # The CNI network configuration to install on each node.
+  cni_network_config: |-
+    {
+        "name": "k8s-pod-network",
+        "cniVersion": "0.3.0",
+        "type": "calico",
+        "log_level": "debug",
+        "datastore_type": "kubernetes",
+        "nodename": "__KUBERNETES_NODE_NAME__",
+        "mtu": ${network_mtu},
+        "ipam": {
+            "type": "host-local",
+            "subnet": "usePodCidr"
+        },
+        "policy": {
+            "type": "k8s",
+            "k8s_auth_token": "__SERVICEACCOUNT_TOKEN__"
+        },
+        "kubernetes": {
+            "k8s_api_root": "https://__KUBERNETES_SERVICE_HOST__:__KUBERNETES_SERVICE_PORT__",
+            "kubeconfig": "__KUBECONFIG_FILEPATH__"
+        }
+    }

resources/calico/calico-gloabl-felix-configs.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Felix Configuration
+kind: CustomResourceDefinition
+metadata:
+   name: globalfelixconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalFelixConfig
+    plural: globalfelixconfigs
+    singular: globalfelixconfig

resources/calico/calico-global-bgp-configs.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global BGP Configuration
+kind: CustomResourceDefinition
+metadata:
+  name: globalbgpconfigs.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalBGPConfig
+    plural: globalbgpconfigs
+    singular: globalbgpconfig

resources/calico/calico-ip-pools.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico IP Pools
+kind: CustomResourceDefinition
+metadata:
+  name: ippools.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: IPPool
+    plural: ippools
+    singular: ippool

resources/calico/calico-network-policies.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworkpolicies.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkPolicy
+    plural: globalnetworkpolicies
+    singular: globalnetworkpolicy

resources/calico/calico-service-account.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: calico-node
+  namespace: kube-system

resources/calico/calico.yaml

@@ -0,0 +1,138 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: calico-node
+  namespace: kube-system
+  labels:
+    k8s-app: calico-node
+spec:
+  selector:
+    matchLabels:
+      k8s-app: calico-node
+  template:
+    metadata:
+      labels:
+        k8s-app: calico-node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      serviceAccountName: calico-node
+      tolerations:
+        # Allow the pod to run on master nodes
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+        # Mark the pod as a critical add-on for rescheduling
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+      containers:
+        - name: calico-node
+          image: quay.io/calico/node:v2.5.1
+          env:
+            # Use Kubernetes API as the backing datastore.
+            - name: DATASTORE_TYPE
+              value: "kubernetes"
+            # Enable felix info logging.
+            - name: FELIX_LOGSEVERITYSCREEN
+              value: "info"
+            # Cluster type to identify the deployment type
+            - name: CLUSTER_TYPE
+              value: "k8s,bgp"
+            # Disable file logging so `kubectl logs` works.
+            - name: CALICO_DISABLE_FILE_LOGGING
+              value: "true"
+            # Set Felix endpoint to host default action to ACCEPT.
+            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
+              value: "ACCEPT"
+            # Disable IPV6 on Kubernetes.
+            - name: FELIX_IPV6SUPPORT
+              value: "false"
+            # Set MTU for tunnel device used if ipip is enabled
+            - name: FELIX_IPINIPMTU
+              value: "${network_mtu}"
+            # Wait for the datastore.
+            - name: WAIT_FOR_DATASTORE
+              value: "true"
+            # The Calico IPv4 pool CIDR (should match `--cluster-cidr`).
+            - name: CALICO_IPV4POOL_CIDR
+              value: "${pod_cidr}"
+            # Enable IPIP
+            - name: CALICO_IPV4POOL_IPIP
+              value: "always"
+            # Enable IP-in-IP within Felix.
+            - name: FELIX_IPINIPENABLED
+              value: "true"
+            # Set node name based on k8s nodeName.
+            - name: NODENAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: FELIX_HEALTHENABLED
+              value: "true"
+          securityContext:
+            privileged: true
+          resources:
+            requests:
+              cpu: 250m
+          livenessProbe:
+            httpGet:
+              path: /liveness
+              port: 9099
+            periodSeconds: 10
+            initialDelaySeconds: 10
+            failureThreshold: 6
+          readinessProbe:
+            httpGet:
+              path: /readiness
+              port: 9099
+            periodSeconds: 10
+          volumeMounts:
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - mountPath: /var/run/calico
+              name: var-run-calico
+              readOnly: false
+        # Install Calico CNI binaries and CNI network config file on nodes
+        - name: install-cni
+          image: quay.io/calico/cni:v1.10.0
+          command: ["/install-cni.sh"]
+          env:
+            - name: CNI_NETWORK_CONFIG
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: cni_network_config
+            - name: CNI_NET_DIR
+              value: "/etc/kubernetes/cni/net.d"
+            # Set node name based on k8s nodeName
+            - name: KUBERNETES_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+      volumes:
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: var-run-calico
+          hostPath:
+            path: /var/run/calico
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/kubernetes/cni/net.d
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

resources/etcd/migrate-etcd-cluster.json

@@ -1,6 +1,6 @@
 {
-  "apiVersion": "etcd.coreos.com/v1beta1",
-  "kind": "Cluster",
+  "apiVersion": "etcd.database.coreos.com/v1beta2",
+  "kind": "EtcdCluster",
   "metadata": {
     "name": "kube-etcd",
     "namespace": "kube-system"
@@ -26,10 +26,10 @@
     "TLS": {
       "static": {
         "member": {
-          "peerSecret": "etcd-member-peer-tls",
-          "clientSecret": "etcd-member-client-tls"
+          "peerSecret": "etcd-peer-tls",
+          "serverSecret": "etcd-server-tls"
         },
-        "operatorSecret": "etcd-operator-client-tls"
+        "operatorSecret": "etcd-client-tls"
       }
     }
   }

resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml

@@ -21,13 +21,13 @@ spec:
     - --initial-cluster-state=new
     - --data-dir=/var/etcd/data
     - --peer-client-cert-auth=true
-    - --peer-trusted-ca-file=/etc/kubernetes/secrets/etcdMember/peer-ca-crt.pem
-    - --peer-cert-file=/etc/kubernetes/secrets/etcdMember/peer-crt.pem
-    - --peer-key-file=/etc/kubernetes/secrets/etcdMember/peer-key.pem
+    - --peer-trusted-ca-file=/etc/kubernetes/secrets/etcd/peer-ca.crt
+    - --peer-cert-file=/etc/kubernetes/secrets/etcd/peer.crt
+    - --peer-key-file=/etc/kubernetes/secrets/etcd/peer.key
     - --client-cert-auth=true
-    - --trusted-ca-file=/etc/kubernetes/secrets/etcdMember/client-ca-crt.pem
-    - --cert-file=/etc/kubernetes/secrets/etcdMember/client-crt.pem
-    - --key-file=/etc/kubernetes/secrets/etcdMember/client-key.pem
+    - --trusted-ca-file=/etc/kubernetes/secrets/etcd/server-ca.crt
+    - --cert-file=/etc/kubernetes/secrets/etcd/server.crt
+    - --key-file=/etc/kubernetes/secrets/etcd/server.key
     volumeMounts:
       - mountPath: /etc/kubernetes/secrets
         name: secrets

resources/experimental/manifests/etcd-client-tls.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-client-tls
+  namespace: kube-system
+type: Opaque
+data:
+  etcd-client-ca.crt: ${etcd_ca_cert}
+  etcd-client.crt: ${etcd_client_cert}
+  etcd-client.key: ${etcd_client_key}

resources/experimental/manifests/etcd-member-client-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-member-client-tls
-  namespace: kube-system
-type: Opaque
-data:
-  client-ca-crt.pem: ${etcd_ca_cert}
-  client-crt.pem: ${etcd_client_cert}
-  client-key.pem: ${etcd_client_key}

resources/experimental/manifests/etcd-member-peer-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-member-peer-tls
-  namespace: kube-system
-type: Opaque
-data:
-  peer-ca-crt.pem: ${etcd_ca_cert}
-  peer-crt.pem: ${etcd_peer_cert}
-  peer-key.pem: ${etcd_peer_key}

resources/experimental/manifests/etcd-operator-client-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-operator-client-tls
-  namespace: kube-system
-type: Opaque
-data:
-  etcd-ca-crt.pem: ${etcd_ca_cert}
-  etcd-crt.pem: ${etcd_client_cert}
-  etcd-key.pem: ${etcd_client_key}

resources/experimental/manifests/etcd-operator.yaml

@@ -19,10 +19,10 @@ spec:
     spec:
       containers:
       - name: etcd-operator
-        image: quay.io/coreos/etcd-operator:v0.3.3
+        image: quay.io/coreos/etcd-operator:v0.5.0
         command:
-          - /usr/local/bin/etcd-operator
-          - --analytics=false
+        - /usr/local/bin/etcd-operator
+        - --analytics=false
         env:
         - name: MY_POD_NAMESPACE
           valueFrom:

resources/experimental/manifests/etcd-peer-tls.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-peer-tls
+  namespace: kube-system
+type: Opaque
+data:
+  peer-ca.crt: ${etcd_ca_cert}
+  peer.crt: ${etcd_peer_cert}
+  peer.key: ${etcd_peer_key}

resources/experimental/manifests/etcd-server-tls.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: etcd-server-tls
+  namespace: kube-system
+type: Opaque
+data:
+  server-ca.crt: ${etcd_ca_cert}
+  server.crt: ${etcd_server_cert}
+  server.key: ${etcd_server_key}

resources/experimental/manifests/etcd-service.yaml

@@ -3,6 +3,10 @@ kind: Service
 metadata:
   name: etcd-service
   namespace: kube-system
+  # This alpha annotation will retain the endpoints even if the etcd pod isn't ready.
+  # This feature is always enabled in endpoint controller in k8s even it is alpha.
+  annotations:
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
   selector:
     app: etcd

resources/flannel/kube-flannel-cfg.yaml

@@ -12,7 +12,8 @@ data:
       "name": "cbr0",
       "type": "flannel",
       "delegate": {
-        "isDefaultGateway": true
+        "isDefaultGateway": true,
+        "hairpinMode": true
       }
     }
   net-conf.json: |

resources/flannel/kube-flannel.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.7.1-amd64
+        image: quay.io/coreos/flannel:v0.8.0-amd64
         command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr", "--iface=$(POD_IP)"]
         securityContext:
           privileged: true
@@ -40,7 +40,7 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       - name: install-cni
-        image: quay.io/coreos/flannel-cni:0.1.0
+        image: quay.io/coreos/flannel-cni:v0.2.0
         command: ["/install-cni.sh"]
         env:
         - name: CNI_NETWORK_CONFIG

resources/manifests/kube-apiserver-secret.yaml

@@ -9,6 +9,6 @@ data:
   apiserver.crt: ${apiserver_cert}
   service-account.pub: ${serviceaccount_pub}
   ca.crt: ${ca_cert}
-  etcd-ca.crt: ${etcd_ca_cert}
+  etcd-client-ca.crt: ${etcd_ca_cert}
   etcd-client.crt: ${etcd_client_cert}
   etcd-client.key: ${etcd_client_key}

resources/manifests/kube-apiserver.yaml

@@ -32,9 +32,10 @@ spec:
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider=${cloud_provider}
-        - --etcd-cafile=/etc/kubernetes/secrets/etcd-ca.crt 
+        - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
         - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
         - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
+        - --etcd-quorum-read=true
         - --etcd-servers=${etcd_servers}
         - --insecure-port=0
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
@@ -81,6 +82,6 @@ spec:
         hostPath:
           path: /var/lock
   updateStrategy:
-      rollingUpdate:
-        maxUnavailable: 1
-      type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+    type: RollingUpdate

resources/manifests/kube-controller-manager.yaml

@@ -30,7 +30,7 @@ spec:
                 - key: k8s-app
                   operator: In
                   values:
-                  - kube-contoller-manager
+                  - kube-controller-manager
               topologyKey: kubernetes.io/hostname
       containers:
       - name: kube-controller-manager

resources/manifests/kube-dns-deployment.yaml

@@ -6,6 +6,7 @@ metadata:
   labels:
     k8s-app: kube-dns
     kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   # replicas: not specified here:
   # 1. In order to make Addon Manager do not reconcile this replicas parameter.
@@ -25,9 +26,22 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+      - name: kube-dns-config
+        configMap:
+          name: kube-dns
+          optional: true
       containers:
       - name: kubedns
-        image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1
+        image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.4
         resources:
           # TODO: Set memory limits when we've profiled the container for large
           # clusters, then set request = limit to keep this container in
@@ -78,7 +92,7 @@ spec:
         - name: kube-dns-config
           mountPath: /kube-dns-config
       - name: dnsmasq
-        image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.1
+        image: gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.4
         livenessProbe:
           httpGet:
             path: /healthcheck/dnsmasq
@@ -116,7 +130,7 @@ spec:
         - name: kube-dns-config
           mountPath: /etc/k8s/dns/dnsmasq-nanny
       - name: sidecar
-        image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.1
+        image: gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.4
         livenessProbe:
           httpGet:
             path: /metrics
@@ -140,16 +154,3 @@ spec:
             memory: 20Mi
             cpu: 10m
       dnsPolicy: Default  # Don't use cluster DNS.
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: kube-dns-config
-        configMap:
-          name: kube-dns
-          optional: true

resources/manifests/kube-etcd-network-checkpointer.yaml

@@ -16,7 +16,7 @@ spec:
         checkpointer.alpha.coreos.com/checkpoint: "true"
     spec:
       containers:
-      - image: quay.io/coreos/kenc:8f6e2e885f790030fbbb0496ea2a2d8830e58b8f
+      - image: quay.io/coreos/kenc:0.0.2
         name: kube-etcd-network-checkpointer
         securityContext:
           privileged: true

resources/manifests/pod-checkpointer.yaml

@@ -17,7 +17,7 @@ spec:
     spec:
       containers:
       - name: pod-checkpointer
-        image: quay.io/coreos/pod-checkpointer:4e7a7dab10bc4d895b66c21656291c6e0b017248
+        image: quay.io/coreos/pod-checkpointer:0cd390e0bc1dcdcc714b20eda3435c3d00669d0e
         command:
         - /checkpoint
         - --v=4

terraform.tfvars.example

@@ -2,4 +2,5 @@ cluster_name = "example"
 api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
 asset_dir = "/home/core/mycluster"
+networking = "flannel"
 experimental_self_hosted_etcd = false

tls-etcd.tf

@@ -1,7 +1,7 @@
-# etcd-ca.crt
-resource "local_file" "etcd_ca_crt" {
+# etcd-client-ca.crt
+resource "local_file" "etcd_client_ca_crt" {
   content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-ca.crt"
+  filename = "${var.asset_dir}/tls/etcd-client-ca.crt"
 }
 
 # etcd-client.crt
@@ -16,72 +16,40 @@ resource "local_file" "etcd_client_key" {
   filename = "${var.asset_dir}/tls/etcd-client.key"
 }
 
-# etcd-peer.crt
+# server-ca.crt
+resource "local_file" "etcd_server_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server-ca.crt"
+}
+
+# server.crt
+resource "local_file" "etcd_server_crt" {
+  content  = "${tls_locally_signed_cert.server.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server.crt"
+}
+
+# server.key
+resource "local_file" "etcd_server_key" {
+  content  = "${tls_private_key.server.private_key_pem}"
+  filename = "${var.asset_dir}/tls/etcd/server.key"
+}
+
+# peer-ca.crt
+resource "local_file" "etcd_peer_ca_crt" {
+  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
+  filename = "${var.asset_dir}/tls/etcd/peer-ca.crt"
+}
+
+# peer.crt
 resource "local_file" "etcd_peer_crt" {
   content  = "${tls_locally_signed_cert.peer.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcd-peer.crt"
+  filename = "${var.asset_dir}/tls/etcd/peer.crt"
 }
 
-# etcd-peer.key
+# peer.key
 resource "local_file" "etcd_peer_key" {
   content  = "${tls_private_key.peer.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcd-peer.key"
-}
-
-# add certs / keys for self-hosted etcd
-
-# operator/etcd-ca-crt.pem
-resource "local_file" "etcd_operator_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/operator/etcd-ca-crt.pem"
-}
-
-# operator/etcd-crt.pem
-resource "local_file" "etcd_operator_client_crt" {
-  content  = "${tls_locally_signed_cert.client.cert_pem}"
-  filename = "${var.asset_dir}/tls/operator/etcd-crt.pem"
-}
-
-# operator/etcd-key.pem
-resource "local_file" "etcd_operator_client_key" {
-  content  = "${tls_private_key.client.private_key_pem}"
-  filename = "${var.asset_dir}/tls/operator/etcd-key.pem"
-}
-
-# etcdMember/client-ca-crt.pem
-resource "local_file" "etcd_member_client_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/client-ca-crt.pem"
-}
-
-# etcdMember/client-crt.pem
-resource "local_file" "etcd_member_client_crt" {
-  content  = "${tls_locally_signed_cert.client.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/client-crt.pem"
-}
-
-# etcdMember/client-key.pem
-resource "local_file" "etcd_member_client_key" {
-  content  = "${tls_private_key.client.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/client-key.pem"
-}
-
-# etcdMember/peer-ca-crt.pem
-resource "local_file" "etcd_member_peer_ca_crt" {
-  content  = "${tls_self_signed_cert.etcd-ca.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/peer-ca-crt.pem"
-}
-
-# etcdMember/peer-crt.pem
-resource "local_file" "etcd_member_peer_crt" {
-  content  = "${tls_locally_signed_cert.peer.cert_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/peer-crt.pem"
-}
-
-# etcdMember/peer-key.pem
-resource "local_file" "etcd_member_peer_key" {
-  content  = "${tls_private_key.peer.private_key_pem}"
-  filename = "${var.asset_dir}/tls/etcdMember/peer-key.pem"
+  filename = "${var.asset_dir}/tls/etcd/peer.key"
 }
 
 # certificates and keys
@@ -110,6 +78,8 @@ resource "tls_self_signed_cert" "etcd-ca" {
   ]
 }
 
+# client certs are used for client (apiserver, locksmith, etcd-operator)
+# to etcd communication
 resource "tls_private_key" "client" {
   algorithm = "RSA"
   rsa_bits  = "2048"
@@ -156,6 +126,52 @@ resource "tls_locally_signed_cert" "client" {
   ]
 }
 
+resource "tls_private_key" "server" {
+  algorithm = "RSA"
+  rsa_bits  = "2048"
+}
+
+resource "tls_cert_request" "server" {
+  key_algorithm   = "${tls_private_key.server.algorithm}"
+  private_key_pem = "${tls_private_key.server.private_key_pem}"
+
+  subject {
+    common_name  = "etcd-server"
+    organization = "etcd"
+  }
+
+  ip_addresses = [
+    "127.0.0.1",
+    "${cidrhost(var.service_cidr, 15)}",
+    "${cidrhost(var.service_cidr, 20)}",
+  ]
+
+  dns_names = "${concat(
+    var.etcd_servers,
+    list(
+      "localhost",
+      "*.kube-etcd.kube-system.svc.cluster.local",
+      "kube-etcd-client.kube-system.svc.cluster.local",
+    ))}"
+}
+
+resource "tls_locally_signed_cert" "server" {
+  cert_request_pem = "${tls_cert_request.server.cert_request_pem}"
+
+  ca_key_algorithm   = "${join(" ", tls_self_signed_cert.etcd-ca.*.key_algorithm)}"
+  ca_private_key_pem = "${join(" ", tls_private_key.etcd-ca.*.private_key_pem)}"
+  ca_cert_pem        = "${join(" ", tls_self_signed_cert.etcd-ca.*.cert_pem)}"
+
+  validity_period_hours = 8760
+
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature",
+    "server_auth",
+    "client_auth",
+  ]
+}
+
 resource "tls_private_key" "peer" {
   algorithm = "RSA"
   rsa_bits  = "2048"
@@ -169,17 +185,17 @@ resource "tls_cert_request" "peer" {
     common_name  = "etcd-peer"
     organization = "etcd"
   }
-  
+
+  ip_addresses = [
+    "${cidrhost(var.service_cidr, 20)}",
+  ]
+
   dns_names = "${concat(
     var.etcd_servers,
     list(
       "*.kube-etcd.kube-system.svc.cluster.local",
       "kube-etcd-client.kube-system.svc.cluster.local",
     ))}"
-
-  ip_addresses = [
-    "${cidrhost(var.service_cidr, 20)}"
-  ]
 }
 
 resource "tls_locally_signed_cert" "peer" {

variables.tf

@@ -29,6 +29,18 @@ variable "cloud_provider" {
   default     = ""
 }
 
+variable "networking" {
+  description = "Choice of networking provider (flannel or calico)"
+  type        = "string"
+  default     = "flannel"
+}
+
+variable "network_mtu" {
+  description = "CNI interface MTU (applies to calico only)"
+  type        = "string"
+  default     = "1500"
+}
+
 variable "pod_cidr" {
   description = "CIDR IP range to assign Kubernetes pods"
   type        = "string"
@@ -50,7 +62,7 @@ variable "container_images" {
   type        = "map"
 
   default = {
-    hyperkube = "quay.io/coreos/hyperkube:v1.6.6_coreos.1"
+    hyperkube = "quay.io/coreos/hyperkube:v1.7.5_coreos.0"
     etcd      = "quay.io/coreos/etcd:v3.1.8"
   }
 }