Merge pull request #411 from coreos/self-hosted-flannel

examples: Update Kubernetes to use self-hosted flannel

CHANGES.md

@@ -27,6 +27,7 @@ Rename `bootcfg` to CoreOS `matchbox`!
 
 * Upgrade Kubernetes v1.5.1 (static) example clusters
 * Upgrade Kubernetes v1.5.1 (self-hosted) example cluster
+* Switch Kubernetes (self-hosted) to run flannel as pods
 * Combine rktnetes Ignition into Kubernetes static cluster
 
 #### Migration

Documentation/bootkube.md

@@ -12,12 +12,12 @@ Ensure that you've gone through the [matchbox with rkt](getting-started-rkt.md)
 * Create the example libvirt client VMs
 * `/etc/hosts` entries for `node[1-3].example.com` (or pass custom names to `k8s-certgen`)
 
-Install [bootkube](https://github.com/kubernetes-incubator/bootkube/releases/tag/v0.3.1) v0.3.1 and add it somewhere on your PATH.
+Install [bootkube](https://github.com/kubernetes-incubator/bootkube/releases/tag/v0.3.4) v0.3.4 and add it somewhere on your PATH.
 
-    $ wget https://github.com/kubernetes-incubator/bootkube/releases/download/v0.3.1/bootkube.tar.gz
+    $ wget https://github.com/kubernetes-incubator/bootkube/releases/download/v0.3.4/bootkube.tar.gz
     $ tar xzf bootkube.tar.gz
     $ ./bin/linux/bootkube version
-    Version: v0.3.1
+    Version: v0.3.4
 
 ## Examples
 
@@ -91,17 +91,23 @@ You may cleanup the `bootkube` assets on the node, but you should keep the copy
 
     $ kubectl get pods --all-namespaces
     NAMESPACE     NAME                                       READY     STATUS    RESTARTS   AGE
-    kube-system   checkpoint-installer-cpjrm                 1/1       Running   0          2m
-    kube-system   kube-apiserver-rvjes                       1/1       Running   0          2m
-    kube-system   kube-controller-manager-3900529476-5n9xb   1/1       Running   0          4m
-    kube-system   kube-controller-manager-3900529476-rq6p8   1/1       Running   0          4m
-    kube-system   kube-dns-4101612645-oeu5g                  4/4       Running   0          4m
-    kube-system   kube-proxy-f5kb8                           1/1       Running   0          2m
-    kube-system   kube-proxy-jkg4z                           1/1       Running   0          2m
-    kube-system   kube-proxy-rrmuv                           1/1       Running   0          2m
-    kube-system   kube-scheduler-1084603659-qaqbk            1/1       Running   0          3m
-    kube-system   kube-scheduler-1084603659-rztvw            1/1       Running   0          4m
-    kube-system   pod-checkpointer-node1.example.com         1/1       Running   0          1m
+    kube-system   checkpoint-installer-p8g8r                 1/1       Running   1          13m
+    kube-system   kube-apiserver-s5gnx                       1/1       Running   1          41s
+    kube-system   kube-controller-manager-3438979800-jrlnd   1/1       Running   1          13m
+    kube-system   kube-controller-manager-3438979800-tkjx7   1/1       Running   1          13m
+    kube-system   kube-dns-4101612645-xt55f                  4/4       Running   4          13m
+    kube-system   kube-flannel-pl5c2                         2/2       Running   0          13m
+    kube-system   kube-flannel-r9t5r                         2/2       Running   3          13m
+    kube-system   kube-flannel-vfb0s                         2/2       Running   4          13m
+    kube-system   kube-proxy-cvhmj                           1/1       Running   0          13m
+    kube-system   kube-proxy-hf9mh                           1/1       Running   1          13m
+    kube-system   kube-proxy-kpl73                           1/1       Running   1          13m
+    kube-system   kube-scheduler-694795526-1l23b             1/1       Running   1          13m
+    kube-system   kube-scheduler-694795526-fks0b             1/1       Running   1          13m
+    kube-system   pod-checkpointer-node1.example.com         1/1       Running   2          10m
+
+
+
 
 Try deleting pods to see that the cluster is resilient to failures and machine restarts (CoreOS auto-updates).
 

examples/ignition/bootkube-controller.yaml

@@ -14,21 +14,8 @@ systemd:
             Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380"
             Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}"
             Environment="ETCD_STRICT_RECONFIG_CHECK=true"
-    - name: flanneld.service
-      enable: true
-      dropins:
-        - name: 40-ExecStartPre-symlink.conf
-          contents: |
-            [Service]
-            ExecStartPre=/opt/init-flannel
     - name: docker.service
       enable: true
-      dropins:
-        - name: 40-flannel.conf
-          contents: |
-            [Unit]
-            Requires=flanneld.service
-            After=flanneld.service
     - name: locksmithd.service
       dropins:
         - name: 40-etcd-lock.conf
@@ -48,22 +35,26 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet via Hyperkube ACI
-        Wants=flanneld.service
         [Service]
         Environment="RKT_OPTS=--uuid-file-save=/var/run/kubelet-pod.uuid \
           --volume=resolv,kind=host,source=/etc/resolv.conf \
           --mount volume=resolv,target=/etc/resolv.conf \
+          --volume var-lib-cni,kind=host,source=/var/lib/cni \
+          --mount volume=var-lib-cni,target=/var/lib/cni \
           --volume var-log,kind=host,source=/var/log \
           --mount volume=var-log,target=/var/log"
         EnvironmentFile=/etc/kubernetes/kubelet.env
-        ExecStartPre=/usr/bin/systemctl is-active flanneld.service
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /var/lib/cni
         ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
           --kubeconfig=/etc/kubernetes/kubeconfig \
           --require-kubeconfig \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --network-plugin=cni \
           --lock-file=/var/run/lock/kubelet.lock \
           --exit-on-lock-contention \
           --pod-manifest-path=/etc/kubernetes/manifests \
@@ -133,7 +124,7 @@ storage:
           # Wrapper for bootkube start
           set -e
           BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}"
-          BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.3.1}"
+          BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.3.4}"
           BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/home/core/assets}"
           exec /usr/bin/rkt run \
             --trust-keys-from-https \
@@ -141,35 +132,6 @@ storage:
             --mount volume=assets,target=/assets \
             $RKT_OPTS \
             ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} --net=host --exec=/bootkube -- start --asset-dir=/assets --etcd-server=http://127.0.0.1:2379 "$@"
-    - path: /opt/init-flannel
-      filesystem: root
-      mode: 0544
-      contents:
-        inline: |
-          #!/bin/bash
-          function init_flannel {
-              echo "Waiting for etcd..."
-              while true
-              do
-                  IFS=',' read -ra ES <<< "{{.k8s_etcd_endpoints}}"
-                  for ETCD in "${ES[@]}"; do
-                      echo "Trying: $ETCD"
-                      if [ -n "$(curl --silent "$ETCD/v2/machines")" ]; then
-                          local ACTIVE_ETCD=$ETCD
-                          break
-                      fi
-                      sleep 1
-                  done
-                  if [ -n "$ACTIVE_ETCD" ]; then
-                      break
-                  fi
-              done
-              RES=$(curl --silent -X PUT -d "value={\"Network\":\"{{.k8s_pod_network}}\",\"Backend\":{\"Type\":\"vxlan\"}}" "$ACTIVE_ETCD/v2/keys/coreos.com/network/config?prevExist=false")
-              if [ -z "$(echo $RES | grep '"action":"create"')" ] && [ -z "$(echo $RES | grep 'Key already exists')" ]; then
-                  echo "Unexpected error configuring flannel pod network: $RES"
-              fi
-          }
-          init_flannel
 
 {{ if index . "ssh_authorized_keys" }}
 passwd:

examples/ignition/bootkube-worker.yaml

@@ -10,16 +10,8 @@ systemd:
             Environment="ETCD_PROXY=on"
             Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379"
             Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}"
-    - name: flanneld.service
-      enable: true
     - name: docker.service
       enable: true
-      dropins:
-        - name: 40-flannel.conf
-          contents: |
-            [Unit]
-            Requires=flanneld.service
-            After=flanneld.service
     - name: locksmithd.service
       dropins:
         - name: 40-etcd-lock.conf
@@ -39,22 +31,26 @@ systemd:
       contents: |
         [Unit]
         Description=Kubelet via Hyperkube ACI
-        Wants=flanneld.service
         [Service]
         Environment="RKT_OPTS=--uuid-file-save=/var/run/kubelet-pod.uuid \
           --volume=resolv,kind=host,source=/etc/resolv.conf \
           --mount volume=resolv,target=/etc/resolv.conf \
+          --volume var-lib-cni,kind=host,source=/var/lib/cni \
+          --mount volume=var-lib-cni,target=/var/lib/cni \
           --volume var-log,kind=host,source=/var/log \
           --mount volume=var-log,target=/var/log"
         EnvironmentFile=/etc/kubernetes/kubelet.env
-        ExecStartPre=/usr/bin/systemctl is-active flanneld.service
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets
+        ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d
+        ExecStartPre=/bin/mkdir -p /var/lib/cni
         ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid
         ExecStart=/usr/lib/coreos/kubelet-wrapper \
           --kubeconfig=/etc/kubernetes/kubeconfig \
           --require-kubeconfig \
+          --cni-conf-dir=/etc/kubernetes/cni/net.d \
+          --network-plugin=cni \
           --lock-file=/var/run/lock/kubelet.lock \
           --exit-on-lock-contention \
           --pod-manifest-path=/etc/kubernetes/manifests \