Update README to correspond to bootkube v0.9.1

* Remove self-hosted etcd service IP out, defunct

README.md

@@ -4,7 +4,7 @@
 
 ## Audience
 
-`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon to create and manage Kubernetes clusters across supported platforms. Use the lower-level bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
+`terraform-render-bootkube` is a low-level component of the [Typhoon](https://github.com/poseidon/typhoon) Kubernetes distribution. Use Typhoon modules to create and manage Kubernetes clusters across supported platforms. Use the bootkube module if you'd like to customize a Kubernetes control plane or build your own distribution.
 
 ## Usage
 
@@ -34,9 +34,7 @@ Find bootkube assets rendered to the `asset_dir` path. That's it.
 
 ### Comparison
 
-Render bootkube assets directly with bootkube v0.8.0.
-
-#### On-host etcd (recommended)
+Render bootkube assets directly with bootkube v0.9.1.
 
 ```sh
 bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --etcd-servers=https://node1.example.com:2379
@@ -50,21 +48,3 @@ mv manifests-networking/* manifests
 popd
 diff -rw assets /home/core/mycluster
 ```
-
-#### Self-hosted etcd (discouraged)
-
-```sh
-bootkube render --asset-dir=assets --api-servers=https://node1.example.com:443 --api-server-alt-names=DNS=node1.example.com --experimental-self-hosted-etcd
-```
-
-Compare assets. Note that experimental must be generated to a separate directory for terraform applies to sync. Move the experimental `bootstrap-manifests` and `manifests` files during deployment.
-
-```sh
-pushd /home/core/mycluster
-mv experimental/bootstrap-manifests/* boostrap-manifests
-mv experimental/manifests/* manifests
-mv manifests-networking/* manifests
-popd
-diff -rw assets /home/core/mycluster
-```
-

assets.tf

@@ -5,7 +5,7 @@ resource "template_dir" "bootstrap-manifests" {
 
   vars {
     hyperkube_image = "${var.container_images["hyperkube"]}"
-    etcd_servers    = "${var.experimental_self_hosted_etcd ? format("https://%s:2379,https://127.0.0.1:12379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
+    etcd_servers    = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
 
     cloud_provider = "${var.cloud_provider}"
     pod_cidr       = "${var.pod_cidr}"
@@ -25,14 +25,16 @@ resource "template_dir" "manifests" {
     kubedns_dnsmasq_image  = "${var.container_images["kubedns_dnsmasq"]}"
     kubedns_sidecar_image  = "${var.container_images["kubedns_sidecar"]}"
 
-    etcd_servers = "${var.experimental_self_hosted_etcd ? format("https://%s:2379", cidrhost(var.service_cidr, 15)) : join(",", formatlist("https://%s:2379", var.etcd_servers))}"
+    etcd_servers = "${join(",", formatlist("https://%s:2379", var.etcd_servers))}"
 
-    cloud_provider      = "${var.cloud_provider}"
-    pod_cidr            = "${var.pod_cidr}"
-    service_cidr        = "${var.service_cidr}"
-    kube_dns_service_ip = "${cidrhost(var.service_cidr, 10)}"
+    cloud_provider        = "${var.cloud_provider}"
+    pod_cidr              = "${var.pod_cidr}"
+    service_cidr          = "${var.service_cidr}"
+    cluster_domain_suffix = "${var.cluster_domain_suffix}"
+    kube_dns_service_ip   = "${cidrhost(var.service_cidr, 10)}"
 
     ca_cert            = "${base64encode(var.ca_certificate == "" ? join(" ", tls_self_signed_cert.kube-ca.*.cert_pem) : var.ca_certificate)}"
+    server             = "${format("https://%s:443", element(var.api_servers, 0))}"
     apiserver_key      = "${base64encode(tls_private_key.apiserver.private_key_pem)}"
     apiserver_cert     = "${base64encode(tls_locally_signed_cert.apiserver.cert_pem)}"
     serviceaccount_pub = "${base64encode(tls_private_key.service-account.public_key_pem)}"

conditional.tf

@@ -1,4 +1,4 @@
-# Assets generated only when experimental self-hosted etcd is enabled
+# Assets generated only when certain options are chosen
 
 resource "template_dir" "flannel-manifests" {
   count           = "${var.networking == "flannel" ? 1 : 0}"
@@ -26,49 +26,3 @@ resource "template_dir" "calico-manifests" {
     pod_cidr    = "${var.pod_cidr}"
   }
 }
-
-# bootstrap-etcd.yaml pod bootstrap-manifest
-resource "template_dir" "experimental-bootstrap-manifests" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/bootstrap-manifests"
-  destination_dir = "${var.asset_dir}/experimental/bootstrap-manifests"
-
-  vars {
-    etcd_image                = "${var.container_images["etcd"]}"
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd subfolder - bootstrap-etcd-service.json and migrate-etcd-cluster.json TPR
-resource "template_dir" "etcd-subfolder" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/etcd"
-  destination_dir = "${var.asset_dir}/etcd"
-
-  vars {
-    bootstrap_etcd_service_ip = "${cidrhost(var.service_cidr, 20)}"
-  }
-}
-
-# etcd-operator deployment and etcd-service manifests
-# etcd client, server, and peer tls secrets
-resource "template_dir" "experimental-manifests" {
-  count           = "${var.experimental_self_hosted_etcd ? 1 : 0}"
-  source_dir      = "${path.module}/resources/experimental/manifests"
-  destination_dir = "${var.asset_dir}/experimental/manifests"
-
-  vars {
-    etcd_operator_image     = "${var.container_images["etcd_operator"]}"
-    etcd_checkpointer_image = "${var.container_images["etcd_checkpointer"]}"
-    etcd_service_ip         = "${cidrhost(var.service_cidr, 15)}"
-
-    # Self-hosted etcd TLS certs / keys
-    etcd_ca_cert     = "${base64encode(tls_self_signed_cert.etcd-ca.cert_pem)}"
-    etcd_client_cert = "${base64encode(tls_locally_signed_cert.client.cert_pem)}"
-    etcd_client_key  = "${base64encode(tls_private_key.client.private_key_pem)}"
-    etcd_server_cert = "${base64encode(tls_locally_signed_cert.server.cert_pem)}"
-    etcd_server_key  = "${base64encode(tls_private_key.server.private_key_pem)}"
-    etcd_peer_cert   = "${base64encode(tls_locally_signed_cert.peer.cert_pem)}"
-    etcd_peer_key    = "${base64encode(tls_private_key.peer.private_key_pem)}"
-  }
-}

outputs.tf

@@ -10,10 +10,6 @@ output "kube_dns_service_ip" {
   value = "${cidrhost(var.service_cidr, 10)}"
 }
 
-output "etcd_service_ip" {
-  value = "${cidrhost(var.service_cidr, 15)}"
-}
-
 output "kubeconfig" {
   value = "${data.template_file.kubeconfig.rendered}"
 }

resources/bootstrap-manifests/bootstrap-apiserver.yaml

@@ -8,8 +8,6 @@ spec:
   - name: kube-apiserver
     image: ${hyperkube_image}
     command:
-    - /usr/bin/flock
-    - /var/lock/api-server.lock
     - /hyperkube
     - apiserver
     - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds

resources/bootstrap-manifests/bootstrap-controller-manager.yaml

@@ -12,6 +12,7 @@ spec:
     - controller-manager
     - --allocate-node-cidrs=true
     - --cluster-cidr=${pod_cidr}
+    - --service-cluster-ip-range=${service_cidr}
     - --cloud-provider=${cloud_provider}
     - --configure-cloud-routes=false
     - --kubeconfig=/etc/kubernetes/kubeconfig

resources/calico/calico.yaml

@@ -13,8 +13,6 @@ spec:
     metadata:
       labels:
         k8s-app: calico-node
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       hostNetwork: true
       serviceAccountName: calico-node
@@ -22,9 +20,6 @@ spec:
         # Allow the pod to run on master nodes
         - key: node-role.kubernetes.io/master
           effect: NoSchedule
-        # Mark the pod as a critical add-on for rescheduling
-        - key: "CriticalAddonsOnly"
-          operator: "Exists"
       containers:
         - name: calico-node
           image: ${calico_image}
@@ -58,7 +53,7 @@ spec:
               value: "${pod_cidr}"
             # Enable IPIP
             - name: CALICO_IPV4POOL_IPIP
-              value: "always"
+              value: "Always"
             # Enable IP-in-IP within Felix.
             - name: FELIX_IPINIPENABLED
               value: "true"
@@ -119,6 +114,7 @@ spec:
               name: cni-bin-dir
             - mountPath: /host/etc/cni/net.d
               name: cni-net-dir
+      terminationGracePeriodSeconds: 0
       volumes:
         - name: lib-modules
           hostPath:

resources/etcd/bootstrap-etcd-service.json

@@ -1,26 +0,0 @@
-{
-  "apiVersion": "v1",
-  "kind": "Service",
-  "metadata": {
-    "name": "bootstrap-etcd-service",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "selector": {
-      "k8s-app": "boot-etcd"
-    },
-    "clusterIP": "${bootstrap_etcd_service_ip}",
-    "ports": [
-      {
-        "name": "client",
-        "port": 12379,
-        "protocol": "TCP"
-      },
-      {
-        "name": "peers",
-        "port": 12380,
-        "protocol": "TCP"
-      }
-    ]
-  }
-}

resources/etcd/migrate-etcd-cluster.json

@@ -1,36 +0,0 @@
-{
-  "apiVersion": "etcd.database.coreos.com/v1beta2",
-  "kind": "EtcdCluster",
-  "metadata": {
-    "name": "kube-etcd",
-    "namespace": "kube-system"
-  },
-  "spec": {
-    "size": 1,
-    "version": "v3.1.8",
-    "pod": {
-      "nodeSelector": {
-        "node-role.kubernetes.io/master": ""
-      },
-      "tolerations": [
-        {
-          "key": "node-role.kubernetes.io/master",
-          "operator": "Exists",
-          "effect": "NoSchedule"
-        }
-      ]
-    },
-    "selfHosted": {
-      "bootMemberClientEndpoint": "https://${bootstrap_etcd_service_ip}:12379"
-    },
-    "TLS": {
-      "static": {
-        "member": {
-          "peerSecret": "etcd-peer-tls",
-          "serverSecret": "etcd-server-tls"
-        },
-        "operatorSecret": "etcd-client-tls"
-      }
-    }
-  }
-}

resources/experimental/bootstrap-manifests/bootstrap-etcd.yaml

@@ -1,41 +0,0 @@
-apiVersion: v1
-kind: Pod
-metadata:
-  name: bootstrap-etcd
-  namespace: kube-system
-  labels:
-    k8s-app: boot-etcd
-spec:
-  containers:
-  - name: etcd
-    image: ${etcd_image}
-    command:
-    - /usr/local/bin/etcd
-    - --name=boot-etcd
-    - --listen-client-urls=https://0.0.0.0:12379
-    - --listen-peer-urls=https://0.0.0.0:12380
-    - --advertise-client-urls=https://${bootstrap_etcd_service_ip}:12379
-    - --initial-advertise-peer-urls=https://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster=boot-etcd=https://${bootstrap_etcd_service_ip}:12380
-    - --initial-cluster-token=bootkube
-    - --initial-cluster-state=new
-    - --data-dir=/var/etcd/data
-    - --peer-client-cert-auth=true
-    - --peer-trusted-ca-file=/etc/kubernetes/secrets/etcd/peer-ca.crt
-    - --peer-cert-file=/etc/kubernetes/secrets/etcd/peer.crt
-    - --peer-key-file=/etc/kubernetes/secrets/etcd/peer.key
-    - --client-cert-auth=true
-    - --trusted-ca-file=/etc/kubernetes/secrets/etcd/server-ca.crt
-    - --cert-file=/etc/kubernetes/secrets/etcd/server.crt
-    - --key-file=/etc/kubernetes/secrets/etcd/server.key
-    volumeMounts:
-      - mountPath: /etc/kubernetes/secrets
-        name: secrets
-        readOnly: true
-  volumes:
-  - name: secrets
-    hostPath:
-      path: /etc/kubernetes/bootstrap-secrets 
-  hostNetwork: true
-  restartPolicy: Never
-  dnsPolicy: ClusterFirstWithHostNet 

resources/experimental/manifests/etcd-client-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-client-tls
-  namespace: kube-system
-type: Opaque
-data:
-  etcd-client-ca.crt: ${etcd_ca_cert}
-  etcd-client.crt: ${etcd_client_cert}
-  etcd-client.key: ${etcd_client_key}

resources/experimental/manifests/etcd-operator.yaml

@@ -1,46 +0,0 @@
-apiVersion: apps/v1beta2
-kind: Deployment
-metadata:
-  name: etcd-operator
-  namespace: kube-system
-  labels:
-    k8s-app: etcd-operator
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      k8s-app: etcd-operator
-  template:
-    metadata:
-      labels:
-        k8s-app: etcd-operator
-    spec:
-      containers:
-      - name: etcd-operator
-        image: ${etcd_operator_image}
-        command:
-        - /usr/local/bin/etcd-operator
-        - --analytics=false
-        env:
-        - name: MY_POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        - name: MY_POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 65534
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      maxUnavailable: 1
-      maxSurge: 1

resources/experimental/manifests/etcd-peer-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-peer-tls
-  namespace: kube-system
-type: Opaque
-data:
-  peer-ca.crt: ${etcd_ca_cert}
-  peer.crt: ${etcd_peer_cert}
-  peer.key: ${etcd_peer_key}

resources/experimental/manifests/etcd-server-tls.yaml

@@ -1,10 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: etcd-server-tls
-  namespace: kube-system
-type: Opaque
-data:
-  server-ca.crt: ${etcd_ca_cert}
-  server.crt: ${etcd_server_cert}
-  server.key: ${etcd_server_key}

resources/experimental/manifests/etcd-service.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: etcd-service
-  namespace: kube-system
-  # This alpha annotation will retain the endpoints even if the etcd pod isn't ready.
-  # This feature is always enabled in endpoint controller in k8s even it is alpha.
-  annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
-spec:
-  selector:
-    app: etcd
-    etcd_cluster: kube-etcd
-  clusterIP: ${etcd_service_ip}
-  ports:
-  - name: client
-    port: 2379
-    protocol: TCP

resources/experimental/manifests/kube-etcd-network-checkpointer.yaml

@@ -1,62 +0,0 @@
-apiVersion: apps/v1beta2
-kind: DaemonSet
-metadata:
-  name: kube-etcd-network-checkpointer
-  namespace: kube-system
-  labels:
-    tier: control-plane
-    k8s-app: kube-etcd-network-checkpointer
-spec:
-  selector:
-    matchLabels:
-      tier: control-plane
-      k8s-app: kube-etcd-network-checkpointer
-  template:
-    metadata:
-      labels:
-        tier: control-plane
-        k8s-app: kube-etcd-network-checkpointer
-      annotations:
-        checkpointer.alpha.coreos.com/checkpoint: "true"
-    spec:
-      containers:
-      - image: ${etcd_checkpointer_image}
-        name: kube-etcd-network-checkpointer
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - mountPath: /etc/kubernetes/selfhosted-etcd
-          name: checkpoint-dir
-          readOnly: false
-        - mountPath: /var/etcd
-          name: etcd-dir
-          readOnly: false
-        - mountPath: /var/lock
-          name: var-lock
-          readOnly: false
-        command:
-        - /usr/bin/flock
-        - /var/lock/kenc.lock
-        - -c
-        - "kenc -r -m iptables && kenc -m iptables"
-      hostNetwork: true
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      tolerations:
-      - key: node-role.kubernetes.io/master
-        operator: Exists
-        effect: NoSchedule
-      volumes:
-      - name: checkpoint-dir
-        hostPath:
-          path: /etc/kubernetes/checkpoint-iptables
-      - name: etcd-dir
-        hostPath:
-          path: /var/etcd
-      - name: var-lock
-        hostPath:
-          path: /var/lock
-  updateStrategy:
-    rollingUpdate:
-      maxUnavailable: 1
-    type: RollingUpdate

resources/flannel/kube-flannel-cfg.yaml

@@ -15,6 +15,7 @@ data:
         {
           "type": "flannel",
           "delegate": {
+            "hairpinMode": true,
             "isDefaultGateway": true
           }
         },

resources/manifests/kube-apiserver.yaml

@@ -18,14 +18,11 @@ spec:
         k8s-app: kube-apiserver
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-apiserver
         image: ${hyperkube_image}
         command:
-        - /usr/bin/flock
-        - /var/lock/api-server.lock
         - /hyperkube
         - apiserver
         - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds
@@ -70,8 +67,6 @@ spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kube-controller-manager.yaml

@@ -17,8 +17,6 @@ spec:
       labels:
         tier: control-plane
         k8s-app: kube-controller-manager
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -45,6 +43,7 @@ spec:
         - --allocate-node-cidrs=true
         - --cloud-provider=${cloud_provider}
         - --cluster-cidr=${pod_cidr}
+        - --service-cluster-ip-range=${service_cidr}
         - --configure-cloud-routes=false
         - --leader-elect=true
         - --root-ca-file=/etc/kubernetes/secrets/ca.crt
@@ -68,8 +67,6 @@ spec:
         runAsNonRoot: true
         runAsUser: 65534
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kube-dns-deployment.yaml

@@ -23,14 +23,10 @@ spec:
     metadata:
       labels:
         k8s-app: kube-dns
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       nodeSelector:
         node-role.kubernetes.io/master: ""
       tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
@@ -71,7 +67,7 @@ spec:
           initialDelaySeconds: 3
           timeoutSeconds: 5
         args:
-        - --domain=cluster.local.
+        - --domain=${cluster_domain_suffix}.
         - --dns-port=10053
         - --config-dir=/kube-dns-config
         - --v=2
@@ -110,8 +106,9 @@ spec:
         - --
         - -k
         - --cache-size=1000
+        - --no-negcache
         - --log-facility=-
-        - --server=/cluster.local/127.0.0.1#10053
+        - --server=/${cluster_domain_suffix}/127.0.0.1#10053
         - --server=/in-addr.arpa/127.0.0.1#10053
         - --server=/ip6.arpa/127.0.0.1#10053
         ports:
@@ -143,8 +140,8 @@ spec:
         args:
         - --v=2
         - --logtostderr
-        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local,5,A
-        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local,5,A
+        - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.${cluster_domain_suffix},5,A
+        - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.${cluster_domain_suffix},5,A
         ports:
         - containerPort: 10054
           name: metrics

resources/manifests/kube-proxy-role-binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-proxy
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:node-proxier # Automatically created system role.
+subjects:
+- kind: ServiceAccount
+  name: kube-proxy
+  namespace: kube-system

resources/manifests/kube-proxy-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: kube-proxy

resources/manifests/kube-proxy.yaml

@@ -16,8 +16,6 @@ spec:
       labels:
         tier: node
         k8s-app: kube-proxy
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       containers:
       - name: kube-proxy
@@ -37,26 +35,31 @@ spec:
         securityContext:
           privileged: true
         volumeMounts:
+        - mountPath: /lib/modules
+          name: lib-modules
+          readOnly: true
         - mountPath: /etc/ssl/certs
           name: ssl-certs-host
           readOnly: true
-        - name: etc-kubernetes
+        - name: kubeconfig
           mountPath: /etc/kubernetes
           readOnly: true
       hostNetwork: true
+      serviceAccountName: kube-proxy
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule
       volumes:
-      - hostPath:
-          path: /usr/share/ca-certificates
-        name: ssl-certs-host
-      - name: etc-kubernetes
+      - name: lib-modules
         hostPath:
-          path: /etc/kubernetes
+          path: /lib/modules
+      - name: ssl-certs-host
+        hostPath:
+          path: /usr/share/ca-certificates
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig-in-cluster
   updateStrategy:
     rollingUpdate:
       maxUnavailable: 1

resources/manifests/kube-scheduler.yaml

@@ -17,8 +17,6 @@ spec:
       labels:
         tier: control-plane
         k8s-app: kube-scheduler
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
       affinity:
         podAntiAffinity:
@@ -55,8 +53,6 @@ spec:
         runAsNonRoot: true
         runAsUser: 65534
       tolerations:
-      - key: CriticalAddonsOnly
-        operator: Exists
       - key: node-role.kubernetes.io/master
         operator: Exists
         effect: NoSchedule

resources/manifests/kubeconfig-in-cluster.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: kubeconfig-in-cluster
+  namespace: kube-system
+stringData:
+  kubeconfig: |
+    apiVersion: v1
+    clusters:
+    - name: local
+      cluster:
+        server: ${server}
+        certificate-authority-data: ${ca_cert}
+    users:
+    - name: service-account
+      user:
+        # Use service account token
+        tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    contexts:
+    - context:
+        cluster: local
+        user: service-account

resources/manifests/pod-checkpointer-role-binding.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: pod-checkpointer
+  namespace: kube-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: pod-checkpointer
+subjects:
+- kind: ServiceAccount
+  name: pod-checkpointer
+  namespace: kube-system

resources/manifests/pod-checkpointer-role.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: pod-checkpointer
+  namespace: kube-system
+rules:
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""] # "" indicates the core API group
+  resources: ["secrets", "configmaps"]
+  verbs: ["get"]

resources/manifests/pod-checkpointer-sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: kube-system
+  name: pod-checkpointer

resources/manifests/pod-checkpointer.yaml

@@ -24,8 +24,8 @@ spec:
         image: ${pod_checkpointer_image}
         command:
         - /checkpoint
-        - --v=4
         - --lock-file=/var/run/lock/pod-checkpointer.lock
+        - --kubeconfig=/etc/checkpointer/kubeconfig
         env:
         - name: NODE_NAME
           valueFrom:
@@ -41,10 +41,13 @@ spec:
               fieldPath: metadata.namespace
         imagePullPolicy: Always
         volumeMounts:
+        - mountPath: /etc/checkpointer
+          name: kubeconfig
         - mountPath: /etc/kubernetes
           name: etc-kubernetes
         - mountPath: /var/run
           name: var-run
+      serviceAccountName: pod-checkpointer
       hostNetwork: true
       nodeSelector:
         node-role.kubernetes.io/master: ""
@@ -54,6 +57,9 @@ spec:
         operator: Exists
         effect: NoSchedule
       volumes:
+      - name: kubeconfig
+        secret:
+          secretName: kubeconfig-in-cluster
       - name: etc-kubernetes
         hostPath:
           path: /etc/kubernetes

terraform.tfvars.example

@@ -3,4 +3,3 @@ api_servers = ["node1.example.com"]
 etcd_servers = ["node1.example.com"]
 asset_dir = "/home/core/mycluster"
 networking = "flannel"
-experimental_self_hosted_etcd = false

tls-etcd.tf

@@ -96,16 +96,12 @@ resource "tls_cert_request" "client" {
 
   ip_addresses = [
     "127.0.0.1",
-    "${cidrhost(var.service_cidr, 15)}",
-    "${cidrhost(var.service_cidr, 20)}",
   ]
 
   dns_names = ["${concat(
     var.etcd_servers,
     list(
       "localhost",
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
     ))}"]
 }
 
@@ -142,16 +138,12 @@ resource "tls_cert_request" "server" {
 
   ip_addresses = [
     "127.0.0.1",
-    "${cidrhost(var.service_cidr, 15)}",
-    "${cidrhost(var.service_cidr, 20)}",
   ]
 
   dns_names = ["${concat(
     var.etcd_servers,
     list(
       "localhost",
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
     ))}"]
 }
 
@@ -186,16 +178,7 @@ resource "tls_cert_request" "peer" {
     organization = "etcd"
   }
 
-  ip_addresses = [
-    "${cidrhost(var.service_cidr, 20)}",
-  ]
-
-  dns_names = ["${concat(
-    var.etcd_servers,
-    list(
-      "*.kube-etcd.kube-system.svc.cluster.local",
-      "kube-etcd-client.kube-system.svc.cluster.local",
-    ))}"]
+  dns_names = ["${var.etcd_servers}"]
 }
 
 resource "tls_locally_signed_cert" "peer" {

tls-k8s.tf

@@ -70,7 +70,7 @@ resource "tls_cert_request" "apiserver" {
     "kubernetes",
     "kubernetes.default",
     "kubernetes.default.svc",
-    "kubernetes.default.svc.cluster.local",
+    "kubernetes.default.svc.${var.cluster_domain_suffix}",
   ]
 
   ip_addresses = [

variables.tf

@@ -9,15 +9,10 @@ variable "api_servers" {
 }
 
 variable "etcd_servers" {
-  description = "List of URLs used to reach etcd servers. Ignored if experimental self-hosted etcd is enabled."
+  description = "List of URLs used to reach etcd servers."
   type        = "list"
 }
 
-variable "experimental_self_hosted_etcd" {
-  description = "(Experimental) Create self-hosted etcd assets"
-  default     = false
-}
-
 variable "asset_dir" {
   description = "Path to a directory where generated assets should be placed (contains secrets)"
   type        = "string"
@@ -50,30 +45,33 @@ variable "pod_cidr" {
 variable "service_cidr" {
   description = <<EOD
 CIDR IP range to assign Kubernetes services.
-The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns, the 15th IP will be reserved for self-hosted etcd, and the 20th IP will be reserved for bootstrap self-hosted etcd.
+The 1st IP will be reserved for kube_apiserver, the 10th IP will be reserved for kube-dns.
 EOD
 
   type    = "string"
   default = "10.3.0.0/24"
 }
 
+variable "cluster_domain_suffix" {
+  description = "Queries for domains with the suffix will be answered by kube-dns"
+  type        = "string"
+  default     = "cluster.local"
+}
+
 variable "container_images" {
   description = "Container images to use"
   type        = "map"
 
   default = {
-    calico            = "quay.io/calico/node:v2.6.1"
-    calico_cni        = "quay.io/calico/cni:v1.11.0"
-    etcd              = "quay.io/coreos/etcd:v3.1.8"
-    etcd_operator     = "quay.io/coreos/etcd-operator:v0.5.0"
-    etcd_checkpointer = "quay.io/coreos/kenc:0.0.2"
-    flannel           = "quay.io/coreos/flannel:v0.8.0-amd64"
-    flannel_cni       = "quay.io/coreos/flannel-cni:v0.3.0"
-    hyperkube         = "quay.io/coreos/hyperkube:v1.8.1_coreos.0"
-    kubedns           = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5"
-    kubedns_dnsmasq   = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
-    kubedns_sidecar   = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
-    pod_checkpointer  = "quay.io/coreos/pod-checkpointer:ec22bec63334befacc2b237ab73b1a8b95b0a654"
+    calico           = "quay.io/calico/node:v2.6.3"
+    calico_cni       = "quay.io/calico/cni:v1.11.1"
+    flannel          = "quay.io/coreos/flannel:v0.9.1-amd64"
+    flannel_cni      = "quay.io/coreos/flannel-cni:v0.3.0"
+    hyperkube        = "gcr.io/google_containers/hyperkube:v1.8.5"
+    kubedns          = "gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.5"
+    kubedns_dnsmasq  = "gcr.io/google_containers/k8s-dns-dnsmasq-nanny-amd64:1.14.5"
+    kubedns_sidecar  = "gcr.io/google_containers/k8s-dns-sidecar-amd64:1.14.5"
+    pod_checkpointer = "quay.io/coreos/pod-checkpointer:08fa021813231323e121ecca7383cc64c4afe888"
   }
 }