Update README to correspond to bootkube v0.12.0

* Remove flags deprecated in Kubernetes v1.10.x
* https://github.com/poseidon/terraform-render-bootkube/pull/50

README.md

@@ -34,7 +34,7 @@ Find bootkube assets rendered to the `asset_dir` path. That's it.
 
 ### Comparison
 
-Render bootkube assets directly with bootkube v0.10.0.
+Render bootkube assets directly with bootkube v0.12.0.
 
 ```sh
 bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379

assets.tf

@@ -10,6 +10,8 @@ resource "template_dir" "bootstrap-manifests" {
     cloud_provider = "${var.cloud_provider}"
     pod_cidr       = "${var.pod_cidr}"
     service_cidr   = "${var.service_cidr}"
+
+    trusted_certs_dir = "${var.trusted_certs_dir}"
   }
 }
 
@@ -32,6 +34,7 @@ resource "template_dir" "manifests" {
     service_cidr          = "${var.service_cidr}"
     cluster_domain_suffix = "${var.cluster_domain_suffix}"
     kube_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
+    trusted_certs_dir     = "${var.trusted_certs_dir}"
 
     ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
     server             = "${format("https://%s:443", element(var.api_servers, 0))}"

outputs.tf

@@ -15,7 +15,7 @@ output "kubeconfig" {
 }
 
 output "user-kubeconfig" {
-  value = "${local_file.user-kubeconfig.filename}"
+  value = "${data.template_file.user-kubeconfig.rendered}"
 }
 
 # etcd TLS assets

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -10,18 +10,16 @@ spec:
     command:
     - /hyperkube
     - apiserver
-    - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ValidatingAdmissionWebhook,ResourceQuota,DefaultTolerationSeconds,MutatingAdmissionWebhook
     - --advertise-address=$(POD_IP)
     - --allow-privileged=true
     - --authorization-mode=RBAC
     - --bind-address=0.0.0.0
     - --client-ca-file=/etc/kubernetes/secrets/ca.crt
+    - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
     - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
     - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
     - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-    - --etcd-quorum-read=true
     - --etcd-servers=${etcd_servers}
-    - --insecure-port=0
     - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
     - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
     - --secure-port=443
@@ -29,7 +27,6 @@ spec:
     - --service-cluster-ip-range=${service_cidr}
     - --cloud-provider=${cloud_provider}
     - --storage-backend=etcd3
-    - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
     - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
     - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
     env:
@@ -54,7 +51,7 @@ spec:
       path: /etc/kubernetes/bootstrap-secrets
   - name: ssl-certs-host
     hostPath:
-      path: /usr/share/ca-certificates
+      path: ${trusted_certs_dir}
   - name: var-lock
     hostPath:
       path: /var/lock

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -33,4 +33,4 @@ spec:
       path: /etc/kubernetes
   - name: ssl-host
     hostPath:
-      path: /usr/share/ca-certificates
+      path: ${trusted_certs_dir}

resources/calico/bgpconfigurations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico BGP Configuration
+kind: CustomResourceDefinition
+metadata:
+  name: bgpconfigurations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: BGPConfiguration
+    plural: bgpconfigurations
+    singular: bgpconfiguration

resources/calico/calico-cluster-role.yaml

@@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: calico-node
-  namespace: kube-system
 rules:
   - apiGroups: [""]
     resources:
@@ -23,6 +22,17 @@ rules:
       - get
       - list
       - watch
+      - patch
+  - apiGroups: [""]
+    resources:
+      - services
+    verbs:
+      - get
+  - apiGroups: [""]
+    resources:
+      - endpoints
+    verbs:
+      - get
   - apiGroups: [""]
     resources:
       - nodes
@@ -41,10 +51,15 @@ rules:
   - apiGroups: ["crd.projectcalico.org"]
     resources:
       - globalfelixconfigs
+      - felixconfigurations
       - bgppeers
       - globalbgpconfigs
+      - bgpconfigurations
       - ippools
       - globalnetworkpolicies
+      - globalnetworksets
+      - networkpolicies
+      - clusterinformations
     verbs:
       - create
       - get

resources/calico/calico-config.yaml

@@ -4,6 +4,8 @@ metadata:
   name: calico-config
   namespace: kube-system
 data:
+  # Disable Typha for now.
+  typha_service_name: "none"
   # The CNI network configuration to install on each node.
   cni_network_config: |-
     {
@@ -31,9 +33,8 @@ data:
         },
         {
           "type": "portmap",
-          "capabilities": {
-            "portMappings": true
-          }
+          "snat": true,
+          "capabilities": {"portMappings": true}
         }
       ]
     }

resources/calico/calico-gloabl-felix-configs.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global Felix Configuration
-kind: CustomResourceDefinition
-metadata:
-   name: globalfelixconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalFelixConfig
-    plural: globalfelixconfigs
-    singular: globalfelixconfig

resources/calico/calico-global-bgp-configs.yaml

@@ -1,13 +0,0 @@
-apiVersion: apiextensions.k8s.io/v1beta1
-description: Calico Global BGP Configuration
-kind: CustomResourceDefinition
-metadata:
-  name: globalbgpconfigs.crd.projectcalico.org
-spec:
-  scope: Cluster
-  group: crd.projectcalico.org
-  version: v1
-  names:
-    kind: GlobalBGPConfig
-    plural: globalbgpconfigs
-    singular: globalbgpconfig

resources/calico/calico.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: calico-node
@@ -10,9 +10,9 @@ spec:
     matchLabels:
       k8s-app: calico-node
   updateStrategy:
+    type: RollingUpdate
     rollingUpdate:
       maxUnavailable: 1
-    type: RollingUpdate
   template:
     metadata:
       labels:
@@ -21,9 +21,10 @@ spec:
       hostNetwork: true
       serviceAccountName: calico-node
       tolerations:
-        # Allow the pod to run on master nodes
-        - key: node-role.kubernetes.io/master
-          effect: NoSchedule
+        - effect: NoSchedule
+          operator: Exists
+        - effect: NoExecute
+          operator: Exists
       containers:
         - name: calico-node
           image: ${calico_image}
@@ -61,15 +62,20 @@ spec:
             # Enable IP-in-IP within Felix.
             - name: FELIX_IPINIPENABLED
               value: "true"
+            # Typha support: controlled by the ConfigMap.
+            - name: FELIX_TYPHAK8SSERVICENAME
+              valueFrom:
+                configMapKeyRef:
+                  name: calico-config
+                  key: typha_service_name
             # Set node name based on k8s nodeName.
             - name: NODENAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            # Auto-detect the BGP IP address.
             - name: IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
+              value: "autodetect"
             - name: FELIX_HEALTHENABLED
               value: "true"
           securityContext:
@@ -96,25 +102,30 @@ spec:
             - mountPath: /var/run/calico
               name: var-run-calico
               readOnly: false
+            - mountPath: /var/lib/calico
+              name: var-lib-calico
+              readOnly: false
         # Install Calico CNI binaries and CNI network config file on nodes
         - name: install-cni
           image: ${calico_cni_image}
           command: ["/install-cni.sh"]
           env:
+            # Name of the CNI config file to create on each node.
             - name: CNI_CONF_NAME
-              value: 10-calico.conflist
+              value: "10-calico.conflist"
+            # Contents of the CNI config to create on each node.
             - name: CNI_NETWORK_CONFIG
               valueFrom:
                 configMapKeyRef:
                   name: calico-config
                   key: cni_network_config
-            - name: CNI_NET_DIR
-              value: "/etc/kubernetes/cni/net.d"
             # Set node name based on k8s nodeName
             - name: KUBERNETES_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            - name: CNI_NET_DIR
+              value: "/etc/kubernetes/cni/net.d"
           volumeMounts:
             - mountPath: /host/opt/cni/bin
               name: cni-bin-dir
@@ -122,12 +133,17 @@ spec:
               name: cni-net-dir
       terminationGracePeriodSeconds: 0
       volumes:
+        # Used by calico/node
         - name: lib-modules
           hostPath:
             path: /lib/modules
         - name: var-run-calico
           hostPath:
             path: /var/run/calico
+        - name: var-lib-calico
+          hostPath:
+            path: /var/lib/calico
+        # Used by install-cni
         - name: cni-bin-dir
           hostPath:
             path: /opt/cni/bin

resources/calico/clusterinformations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Cluster Information
+kind: CustomResourceDefinition
+metadata:
+  name: clusterinformations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: ClusterInformation
+    plural: clusterinformations
+    singular: clusterinformation

resources/calico/felixconfigurations-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Felix Configuration
+kind: CustomResourceDefinition
+metadata:
+   name: felixconfigurations.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: FelixConfiguration
+    plural: felixconfigurations
+    singular: felixconfiguration

resources/calico/globalnetworksets-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Global Network Sets
+kind: CustomResourceDefinition
+metadata:
+  name: globalnetworksets.crd.projectcalico.org
+spec:
+  scope: Cluster
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: GlobalNetworkSet
+    plural: globalnetworksets
+    singular: globalnetworkset

resources/calico/networkpolicies-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+description: Calico Network Policies
+kind: CustomResourceDefinition
+metadata:
+  name: networkpolicies.crd.projectcalico.org
+spec:
+  scope: Namespaced
+  group: crd.projectcalico.org
+  version: v1
+  names:
+    kind: NetworkPolicy
+    plural: networkpolicies
+    singular: networkpolicy

resources/flannel/flannel-cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system

resources/flannel/flannel-cluster-role.yaml

@@ -0,0 +1,24 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch

resources/flannel/flannel-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system

resources/flannel/flannel.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-flannel
@@ -17,6 +17,7 @@ spec:
         tier: node
         k8s-app: flannel
     spec:
+      serviceAccountName: flannel
       containers:
       - name: kube-flannel
         image: ${flannel_image}
@@ -59,9 +60,10 @@ spec:
           mountPath: /host/opt/cni/bin/
       hostNetwork: true
       tolerations:
-      - key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
         operator: Exists
-        effect: NoSchedule
       volumes:
         - name: run
           hostPath:

resources/manifests/kube-apiserver.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-apiserver
@@ -25,7 +25,6 @@ spec:
         command:
         - /hyperkube
         - apiserver
-        - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ValidatingAdmissionWebhook,ResourceQuota,DefaultTolerationSeconds,MutatingAdmissionWebhook
         - --advertise-address=$(POD_IP)
         - --allow-privileged=true
         - --anonymous-auth=false
@@ -33,19 +32,17 @@ spec:
         - --bind-address=0.0.0.0
         - --client-ca-file=/etc/kubernetes/secrets/ca.crt
         - --cloud-provider=${cloud_provider}
+        - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultTolerationSeconds,DefaultStorageClass,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
         - --etcd-cafile=/etc/kubernetes/secrets/etcd-client-ca.crt
         - --etcd-certfile=/etc/kubernetes/secrets/etcd-client.crt
         - --etcd-keyfile=/etc/kubernetes/secrets/etcd-client.key
-        - --etcd-quorum-read=true
         - --etcd-servers=${etcd_servers}
-        - --insecure-port=0
         - --kubelet-client-certificate=/etc/kubernetes/secrets/apiserver.crt
         - --kubelet-client-key=/etc/kubernetes/secrets/apiserver.key
         - --secure-port=443
         - --service-account-key-file=/etc/kubernetes/secrets/service-account.pub
         - --service-cluster-ip-range=${service_cidr}
         - --storage-backend=etcd3
-        - --tls-ca-file=/etc/kubernetes/secrets/ca.crt
         - --tls-cert-file=/etc/kubernetes/secrets/apiserver.crt
         - --tls-private-key-file=/etc/kubernetes/secrets/apiserver.key
         env:
@@ -73,7 +70,7 @@ spec:
       volumes:
       - name: ssl-certs-host
         hostPath:
-          path: /usr/share/ca-certificates
+          path: ${trusted_certs_dir}
       - name: secrets
         secret:
           secretName: kube-apiserver

resources/manifests/kube-controller-manager.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-controller-manager
@@ -47,6 +47,7 @@ spec:
         - --service-cluster-ip-range=${service_cidr}
         - --configure-cloud-routes=false
         - --leader-elect=true
+        - --flex-volume-plugin-dir=/var/lib/kubelet/volumeplugins
         - --root-ca-file=/etc/kubernetes/secrets/ca.crt
         - --service-account-private-key-file=/etc/kubernetes/secrets/service-account.key
         livenessProbe:
@@ -59,6 +60,9 @@ spec:
         - name: secrets
           mountPath: /etc/kubernetes/secrets
           readOnly: true
+        - name: volumeplugins
+          mountPath: /var/lib/kubelet/volumeplugins
+          readOnly: true          
         - name: ssl-host
           mountPath: /etc/ssl/certs
           readOnly: true
@@ -78,5 +82,8 @@ spec:
           secretName: kube-controller-manager
       - name: ssl-host
         hostPath:
-          path: /usr/share/ca-certificates
+          path: ${trusted_certs_dir}
+      - name: volumeplugins
+        hostPath:
+          path: /var/lib/kubelet/volumeplugins          
       dnsPolicy: Default # Don't use cluster DNS.

resources/manifests/kube-dns-deployment.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-dns

resources/manifests/kube-proxy.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: kube-proxy
@@ -47,16 +47,17 @@ spec:
       hostNetwork: true
       serviceAccountName: kube-proxy
       tolerations:
-      - key: node-role.kubernetes.io/master
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
         operator: Exists
-        effect: NoSchedule
       volumes:
       - name: lib-modules
         hostPath:
           path: /lib/modules
       - name: ssl-certs-host
         hostPath:
-          path: /usr/share/ca-certificates
+          path: ${trusted_certs_dir}
       - name: kubeconfig
         configMap:
           name: kubeconfig-in-cluster

resources/manifests/kube-scheduler.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: kube-scheduler

resources/manifests/pod-checkpointer.yaml

@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta2
+apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   name: pod-checkpointer

variables.tf

@@ -63,18 +63,24 @@ variable "container_images" {
   type        = "map"
 
   default = {
-    calico           = "quay.io/calico/node:v2.6.6"
-    calico_cni       = "quay.io/calico/cni:v1.11.2"
-    flannel          = "quay.io/coreos/flannel:v0.9.1-amd64"
+    calico           = "quay.io/calico/node:v3.0.4"
+    calico_cni       = "quay.io/calico/cni:v2.0.1"
+    flannel          = "quay.io/coreos/flannel:v0.10.0-amd64"
     flannel_cni      = "quay.io/coreos/flannel-cni:v0.3.0"
-    hyperkube        = "gcr.io/google_containers/hyperkube:v1.9.2"
-    kubedns          = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.8"
-    kubedns_dnsmasq  = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.8"
-    kubedns_sidecar  = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.8"
-    pod_checkpointer = "quay.io/coreos/pod-checkpointer:08fa021813231323e121ecca7383cc64c4afe888"
+    hyperkube        = "k8s.gcr.io/hyperkube:v1.10.0"
+    kubedns          = "k8s.gcr.io/k8s-dns-kube-dns-amd64:1.14.9"
+    kubedns_dnsmasq  = "k8s.gcr.io/k8s-dns-dnsmasq-nanny-amd64:1.14.9"
+    kubedns_sidecar  = "k8s.gcr.io/k8s-dns-sidecar-amd64:1.14.9"
+    pod_checkpointer = "quay.io/coreos/pod-checkpointer:9dc83e1ab3bc36ca25c9f7c18ddef1b91d4a0558"
   }
 }
 
+variable "trusted_certs_dir" {
+  description = "Path to the directory on cluster nodes where trust TLS certs are kept"
+  type        = "string"
+  default     = "/usr/share/ca-certificates"
+}
+
 variable "ca_certificate" {
   description = "Existing PEM-encoded CA certificate (generated if blank)"
   type        = "string"