Update README to correspond to v0.9.0

* Bug fixes for Calico 2.6.x
https://github.com/projectcalico/calico/releases/tag/v2.6.3
* Bug fixes for cni-plugin (i.e. cni) v1.11.x
https://github.com/projectcalico/cni-plugin/releases/tag/v1.11.1

README.md

@@ -4,7 +4,7 @@
 
 ## Audience
 
-`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon to create and manage Kubernetes clusters across supported platforms. Use the lower-level bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
@@ -34,7 +34,7 @@ Find bootkube assets rendered to the `asset_dir` path. That's it.
 
 ### Comparison
 
-Render bootkube assets directly with bootkube v0.8.0.
+Render bootkube assets directly with bootkube v0.9.0.
 
 #### On-host etcd (recommended)
 
@@ -51,7 +51,7 @@ popd
 diff -rw assets /home/core/mycluster
 ```
 
-#### Self-hosted etcd (discouraged)
+#### Self-hosted etcd (deprecated)
 
 ```sh
 bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd

assets.tf

@@ -33,6 +33,7 @@ resource "template_dir" "manifests" {
     kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
 
     ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
+    server       = "${format("https://%s:443", element(var.api_servers, 0))}"
     apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
     apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
     serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -8,8 +8,6 @@ spec:
   - name: kube-apiserver
     image: ${hyperkube_image}
     command:
-    - /usr/bin/flock
-    - /var/lock/api-server.lock
     - /hyperkube
     - apiserver
     - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds

resources/calico/calico.yaml

@@ -13,8 +13,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-node
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       hostNetwork: true
       serviceAccountName: calico-node
@@ -22,9 +20,6 @@ spec:
         # Allow the pod to run on master nodes
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
-        # Mark the pod as a critical add-on for rescheduling
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
       containers:
         - name: calico-node
           image: ${calico_image}
@@ -58,7 +53,7 @@ spec:
               value: "${pod_cidr}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
-              value: "always"
+              value: "Always"
             # Enable IP-in-IP within Felix.
             - name: FELIX_IPINIPENABLED
               value: "true"
@@ -119,6 +114,7 @@ spec:
               name: cni-bin-dir
             - mountPath: /host/etc/cni/net.d
               name: cni-net-dir
+      terminationGracePeriodSeconds: 0
       volumes:
         - name: lib-modules
           hostPath:

resources/flannel/kube-flannel-cfg.yaml

@@ -15,6 +15,7 @@ data:
         {
           "type": "flannel",
           "delegate": {
+            "hairpinMode": true,
             "isDefaultGateway": true
           }
         },

resources/manifests/kube-apiserver.yaml

@@ -18,14 +18,11 @@ spec:
         k8s-app: kube-apiserver
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-apiserver
         image: ${hyperkube_image}
         command:
-        - /usr/bin/flock
-        - /var/lock/api-server.lock
         - /hyperkube
         - apiserver
         - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
@@ -70,8 +67,6 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kube-controller-manager.yaml

@@ -17,8 +17,6 @@ spec:
       labels:
         tier: control-plane
         k8s-app: kube-controller-manager
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -68,8 +66,6 @@ spec:
         runAsNonRoot: true
         runAsUser: 65534
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kube-dns-deployment.yaml

@@ -23,14 +23,10 @@ spec:
     metadata:
       labels:
         k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
@@ -110,6 +106,7 @@ spec:
         - --
         - -k
         - --cache-size=1000
+        - --no-negcache
         - --log-facility=-
         - --server=/cluster.local/127.0.0.1#10053
         - --server=/in-addr.arpa/127.0.0.1#10053

resources/manifests/kube-proxy-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-proxy
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-proxier # Automatically created system role.
+subjects:
+- kind: ServiceAccount
+  name: kube-proxy
+  namespace: kube-system

resources/manifests/kube-proxy-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: kube-proxy

resources/manifests/kube-proxy.yaml

@@ -16,8 +16,6 @@ spec:
       labels:
         tier: node
         k8s-app: kube-proxy
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-proxy
@@ -37,26 +35,31 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
         - mountPath: /etc/ssl/certs
           name: ssl-certs-host
           readOnly: true
-        - name: etc-kubernetes
+        - name: kubeconfig
           mountPath: /etc/kubernetes
           readOnly: true
       hostNetwork: true
+      serviceAccountName: kube-proxy
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
       volumes:
-      - hostPath:
-          path: /usr/share/ca-certificates
-        name: ssl-certs-host
-      - name: etc-kubernetes
+      - name: lib-modules
         hostPath:
-          path: /etc/kubernetes
+          path: /lib/modules
+      - name: ssl-certs-host
+        hostPath:
+          path: /usr/share/ca-certificates
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig-in-cluster
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 1

resources/manifests/kube-scheduler.yaml

@@ -17,8 +17,6 @@ spec:
       labels:
         tier: control-plane
         k8s-app: kube-scheduler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -55,8 +53,6 @@ spec:
         runAsNonRoot: true
         runAsUser: 65534
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kubeconfig-in-cluster.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kubeconfig-in-cluster
+  namespace: kube-system
+stringData:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - name: local
+      cluster:
+        server: ${server}
+        certificate-authority-data: ${ca_cert}
+    users:
+    - name: service-account
+      user:
+        # Use service account token
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    contexts:
+    - context:
+        cluster: local
+        user: service-account

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: pod-checkpointer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: pod-checkpointer
+subjects:
+- kind: ServiceAccount
+  name: pod-checkpointer
+  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: pod-checkpointer
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["secrets", "configmaps"]
+  verbs: ["get"]

resources/manifests/pod-checkpointer-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: pod-checkpointer

resources/manifests/pod-checkpointer.yaml

@@ -24,8 +24,8 @@ spec:
         image: ${pod_checkpointer_image}
         command:
         - /checkpoint
-        - --v=4
         - --lock-file=/var/run/lock/pod-checkpointer.lock
+        - --kubeconfig=/etc/checkpointer/kubeconfig
         env:
         - name: NODE_NAME
           valueFrom:
@@ -41,10 +41,13 @@ spec:
               fieldPath: metadata.namespace
         imagePullPolicy: Always
         volumeMounts:
+        - mountPath: /etc/checkpointer
+          name: kubeconfig
         - mountPath: /etc/kubernetes
           name: etc-kubernetes
         - mountPath: /var/run
           name: var-run
+      serviceAccountName: pod-checkpointer
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -54,6 +57,9 @@ spec:
         operator: Exists
         effect: NoSchedule
       volumes:
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig-in-cluster
       - name: etc-kubernetes
         hostPath:
           path: /etc/kubernetes

variables.tf

@@ -62,18 +62,18 @@ variable "container_images" {
   type        = "map"
 
   default = {
-    calico            = "quay.io/calico/node:v2.6.1"
-    calico_cni        = "quay.io/calico/cni:v1.11.0"
+    calico            = "quay.io/calico/node:v2.6.3"
+    calico_cni        = "quay.io/calico/cni:v1.11.1"
     etcd              = "quay.io/coreos/etcd:v3.1.8"
     etcd_operator     = "quay.io/coreos/etcd-operator:v0.5.0"
     etcd_checkpointer = "quay.io/coreos/kenc:0.0.2"
-    flannel           = "quay.io/coreos/flannel:v0.8.0-amd64"
+    flannel           = "quay.io/coreos/flannel:v0.9.1-amd64"
     flannel_cni       = "quay.io/coreos/flannel-cni:v0.3.0"
-    hyperkube         = "quay.io/coreos/hyperkube:v1.8.1_coreos.0"
+    hyperkube         = "gcr.io/google_containers/hyperkube:v1.8.4"
     kubedns           = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5"
     kubedns_dnsmasq   = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
     kubedns_sidecar   = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
-    pod_checkpointer  = "quay.io/coreos/pod-checkpointer:ec22bec63334befacc2b237ab73b1a8b95b0a654"
+    pod_checkpointer  = "quay.io/coreos/pod-checkpointer:e22cc0e3714378de92f45326474874eb602ca0ac"
   }
 }