From e54418633c29cebf4df91a8fcd662227ae54aa2a Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 13 Jan 2017 15:29:23 -0800 Subject: [PATCH] examples: Update Kubernetes to use self-hosted flannel * Self-hosted Kubernetes now runs flannel as a daemonset --- CHANGES.md | 1 + Documentation/bootkube.md | 34 ++++++++------ examples/ignition/bootkube-controller.yaml | 52 +++------------------- examples/ignition/bootkube-worker.yaml | 16 +++---- 4 files changed, 34 insertions(+), 69 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 721d3346..4fbc06bb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -27,6 +27,7 @@ Rename `bootcfg` to CoreOS `matchbox`! * Upgrade Kubernetes v1.5.1 (static) example clusters * Upgrade Kubernetes v1.5.1 (self-hosted) example cluster +* Switch Kubernetes (self-hosted) to run flannel as pods * Combine rktnetes Ignition into Kubernetes static cluster #### Migration diff --git a/Documentation/bootkube.md b/Documentation/bootkube.md index 094d61af..19b185a0 100644 --- a/Documentation/bootkube.md +++ b/Documentation/bootkube.md @@ -12,12 +12,12 @@ Ensure that you've gone through the [matchbox with rkt](getting-started-rkt.md) * Create the example libvirt client VMs * `/etc/hosts` entries for `node[1-3].example.com` (or pass custom names to `k8s-certgen`) -Install [bootkube](https://github.com/kubernetes-incubator/bootkube/releases/tag/v0.3.1) v0.3.1 and add it somewhere on your PATH. +Install [bootkube](https://github.com/kubernetes-incubator/bootkube/releases/tag/v0.3.4) v0.3.4 and add it somewhere on your PATH. - $ wget https://github.com/kubernetes-incubator/bootkube/releases/download/v0.3.1/bootkube.tar.gz + $ wget https://github.com/kubernetes-incubator/bootkube/releases/download/v0.3.4/bootkube.tar.gz $ tar xzf bootkube.tar.gz $ ./bin/linux/bootkube version - Version: v0.3.1 + Version: v0.3.4 ## Examples @@ -91,17 +91,23 @@ You may cleanup the `bootkube` assets on the node, but you should keep the copy $ kubectl get pods --all-namespaces NAMESPACE NAME READY STATUS RESTARTS AGE - kube-system checkpoint-installer-cpjrm 1/1 Running 0 2m - kube-system kube-apiserver-rvjes 1/1 Running 0 2m - kube-system kube-controller-manager-3900529476-5n9xb 1/1 Running 0 4m - kube-system kube-controller-manager-3900529476-rq6p8 1/1 Running 0 4m - kube-system kube-dns-4101612645-oeu5g 4/4 Running 0 4m - kube-system kube-proxy-f5kb8 1/1 Running 0 2m - kube-system kube-proxy-jkg4z 1/1 Running 0 2m - kube-system kube-proxy-rrmuv 1/1 Running 0 2m - kube-system kube-scheduler-1084603659-qaqbk 1/1 Running 0 3m - kube-system kube-scheduler-1084603659-rztvw 1/1 Running 0 4m - kube-system pod-checkpointer-node1.example.com 1/1 Running 0 1m + kube-system checkpoint-installer-p8g8r 1/1 Running 1 13m + kube-system kube-apiserver-s5gnx 1/1 Running 1 41s + kube-system kube-controller-manager-3438979800-jrlnd 1/1 Running 1 13m + kube-system kube-controller-manager-3438979800-tkjx7 1/1 Running 1 13m + kube-system kube-dns-4101612645-xt55f 4/4 Running 4 13m + kube-system kube-flannel-pl5c2 2/2 Running 0 13m + kube-system kube-flannel-r9t5r 2/2 Running 3 13m + kube-system kube-flannel-vfb0s 2/2 Running 4 13m + kube-system kube-proxy-cvhmj 1/1 Running 0 13m + kube-system kube-proxy-hf9mh 1/1 Running 1 13m + kube-system kube-proxy-kpl73 1/1 Running 1 13m + kube-system kube-scheduler-694795526-1l23b 1/1 Running 1 13m + kube-system kube-scheduler-694795526-fks0b 1/1 Running 1 13m + kube-system pod-checkpointer-node1.example.com 1/1 Running 2 10m + + + Try deleting pods to see that the cluster is resilient to failures and machine restarts (CoreOS auto-updates). diff --git a/examples/ignition/bootkube-controller.yaml b/examples/ignition/bootkube-controller.yaml index 596b8859..024c958f 100644 --- a/examples/ignition/bootkube-controller.yaml +++ b/examples/ignition/bootkube-controller.yaml @@ -14,21 +14,8 @@ systemd: Environment="ETCD_LISTEN_PEER_URLS=http://0.0.0.0:2380" Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}" Environment="ETCD_STRICT_RECONFIG_CHECK=true" - - name: flanneld.service - enable: true - dropins: - - name: 40-ExecStartPre-symlink.conf - contents: | - [Service] - ExecStartPre=/opt/init-flannel - name: docker.service enable: true - dropins: - - name: 40-flannel.conf - contents: | - [Unit] - Requires=flanneld.service - After=flanneld.service - name: locksmithd.service dropins: - name: 40-etcd-lock.conf @@ -48,22 +35,26 @@ systemd: contents: | [Unit] Description=Kubelet via Hyperkube ACI - Wants=flanneld.service [Service] Environment="RKT_OPTS=--uuid-file-save=/var/run/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --mount volume=var-lib-cni,target=/var/lib/cni \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" EnvironmentFile=/etc/kubernetes/kubelet.env - ExecStartPre=/usr/bin/systemctl is-active flanneld.service ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ --kubeconfig=/etc/kubernetes/kubeconfig \ --require-kubeconfig \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --network-plugin=cni \ --lock-file=/var/run/lock/kubelet.lock \ --exit-on-lock-contention \ --pod-manifest-path=/etc/kubernetes/manifests \ @@ -133,7 +124,7 @@ storage: # Wrapper for bootkube start set -e BOOTKUBE_ACI="${BOOTKUBE_ACI:-quay.io/coreos/bootkube}" - BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.3.1}" + BOOTKUBE_VERSION="${BOOTKUBE_VERSION:-v0.3.4}" BOOTKUBE_ASSETS="${BOOTKUBE_ASSETS:-/home/core/assets}" exec /usr/bin/rkt run \ --trust-keys-from-https \ @@ -141,35 +132,6 @@ storage: --mount volume=assets,target=/assets \ $RKT_OPTS \ ${BOOTKUBE_ACI}:${BOOTKUBE_VERSION} --net=host --exec=/bootkube -- start --asset-dir=/assets --etcd-server=http://127.0.0.1:2379 "$@" - - path: /opt/init-flannel - filesystem: root - mode: 0544 - contents: - inline: | - #!/bin/bash - function init_flannel { - echo "Waiting for etcd..." - while true - do - IFS=',' read -ra ES <<< "{{.k8s_etcd_endpoints}}" - for ETCD in "${ES[@]}"; do - echo "Trying: $ETCD" - if [ -n "$(curl --silent "$ETCD/v2/machines")" ]; then - local ACTIVE_ETCD=$ETCD - break - fi - sleep 1 - done - if [ -n "$ACTIVE_ETCD" ]; then - break - fi - done - RES=$(curl --silent -X PUT -d "value={\"Network\":\"{{.k8s_pod_network}}\",\"Backend\":{\"Type\":\"vxlan\"}}" "$ACTIVE_ETCD/v2/keys/coreos.com/network/config?prevExist=false") - if [ -z "$(echo $RES | grep '"action":"create"')" ] && [ -z "$(echo $RES | grep 'Key already exists')" ]; then - echo "Unexpected error configuring flannel pod network: $RES" - fi - } - init_flannel {{ if index . "ssh_authorized_keys" }} passwd: diff --git a/examples/ignition/bootkube-worker.yaml b/examples/ignition/bootkube-worker.yaml index b9a78db1..b5a41a4f 100644 --- a/examples/ignition/bootkube-worker.yaml +++ b/examples/ignition/bootkube-worker.yaml @@ -10,16 +10,8 @@ systemd: Environment="ETCD_PROXY=on" Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379" Environment="ETCD_INITIAL_CLUSTER={{.etcd_initial_cluster}}" - - name: flanneld.service - enable: true - name: docker.service enable: true - dropins: - - name: 40-flannel.conf - contents: | - [Unit] - Requires=flanneld.service - After=flanneld.service - name: locksmithd.service dropins: - name: 40-etcd-lock.conf @@ -39,22 +31,26 @@ systemd: contents: | [Unit] Description=Kubelet via Hyperkube ACI - Wants=flanneld.service [Service] Environment="RKT_OPTS=--uuid-file-save=/var/run/kubelet-pod.uuid \ --volume=resolv,kind=host,source=/etc/resolv.conf \ --mount volume=resolv,target=/etc/resolv.conf \ + --volume var-lib-cni,kind=host,source=/var/lib/cni \ + --mount volume=var-lib-cni,target=/var/lib/cni \ --volume var-log,kind=host,source=/var/log \ --mount volume=var-log,target=/var/log" EnvironmentFile=/etc/kubernetes/kubelet.env - ExecStartPre=/usr/bin/systemctl is-active flanneld.service ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /srv/kubernetes/manifests ExecStartPre=/bin/mkdir -p /etc/kubernetes/checkpoint-secrets + ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d + ExecStartPre=/bin/mkdir -p /var/lib/cni ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/run/kubelet-pod.uuid ExecStart=/usr/lib/coreos/kubelet-wrapper \ --kubeconfig=/etc/kubernetes/kubeconfig \ --require-kubeconfig \ + --cni-conf-dir=/etc/kubernetes/cni/net.d \ + --network-plugin=cni \ --lock-file=/var/run/lock/kubelet.lock \ --exit-on-lock-contention \ --pod-manifest-path=/etc/kubernetes/manifests \