From ee3445454e70d5a4d89ecd1982cb342bc435a63d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Fri, 14 Jul 2017 14:09:53 -0700 Subject: [PATCH] examples: Switch Kubernetes (non-terraform) to use CLUO * Use the container linux update operator to coordinate reboots * Stop using locksmithd for reboot coordination * etcd TLS assets now only need to be distributed to controller nodes which are etcd peers --- Documentation/bootkube.md | 33 +++++++++---- examples/addons/cluo/update-agent.yaml | 55 +++++++++++++++++++++ examples/addons/cluo/update-operator.yaml | 23 +++++++++ examples/groups/bootkube-install/node1.json | 4 +- examples/groups/bootkube-install/node2.json | 3 +- examples/groups/bootkube-install/node3.json | 3 +- examples/groups/bootkube/node1.json | 1 - examples/groups/bootkube/node2.json | 1 - examples/groups/bootkube/node3.json | 1 - examples/ignition/bootkube-controller.yaml | 10 +--- examples/ignition/bootkube-worker.yaml | 10 +--- 11 files changed, 107 insertions(+), 37 deletions(-) create mode 100644 examples/addons/cluo/update-agent.yaml create mode 100644 examples/addons/cluo/update-operator.yaml diff --git a/Documentation/bootkube.md b/Documentation/bootkube.md index 5dfb09fe..2e276134 100644 --- a/Documentation/bootkube.md +++ b/Documentation/bootkube.md @@ -60,32 +60,32 @@ Client machines should boot and provision themselves. Local client VMs should ne We're ready to use bootkube to create a temporary control plane and bootstrap a self-hosted Kubernetes cluster. -Secure copy the etcd TLS assets to `/etc/ssl/etcd/*` on **every** node. +Secure copy the etcd TLS assets to `/etc/ssl/etcd/*` on **every controller** node. -```bash -for node in 'node1' 'node2' 'node3'; do +```sh +for node in 'node1'; do scp -r assets/tls/etcd-* assets/tls/etcd core@$node.example.com:/home/core/ ssh core@$node.example.com 'sudo mkdir -p /etc/ssl/etcd && sudo mv etcd-* etcd /etc/ssl/etcd/ && sudo chown -R etcd:etcd /etc/ssl/etcd && sudo chmod -R 500 /etc/ssl/etcd/' done ``` -Secure copy the `kubeconfig` to `/etc/kubernetes/kubeconfig` on **every** node which will path activate the `kubelet.service`. +Secure copy the `kubeconfig` to `/etc/kubernetes/kubeconfig` on **every node** to path activate the `kubelet.service`. -```bash +```sh for node in 'node1' 'node2' 'node3'; do scp assets/auth/kubeconfig core@$node.example.com:/home/core/kubeconfig ssh core@$node.example.com 'sudo mv kubeconfig /etc/kubernetes/kubeconfig' done ``` -Secure copy the `bootkube` generated assets to any controller node and run `bootkube-start` (takes ~10 minutes). +Secure copy the `bootkube` generated assets to **any controller** node and run `bootkube-start` (takes ~10 minutes). ```sh scp -r assets core@node1.example.com:/home/core ssh core@node1.example.com 'sudo mv assets /opt/bootkube/assets && sudo systemctl start bootkube' ``` -Optionally watch the Kubernetes control plane bootstrapping with the bootkube temporary api-server. You will see quite a bit of output. +Watch the Kubernetes control plane bootstrapping with the bootkube temporary api-server. You will see quite a bit of output. ```sh $ ssh core@node1.example.com 'journalctl -f -u bootkube' @@ -96,11 +96,11 @@ $ ssh core@node1.example.com 'journalctl -f -u bootkube' [ 299.311743] bootkube[5]: All self-hosted control plane components successfully started ``` -You may cleanup the `bootkube` assets on the node, but you should keep the copy on your laptop. It contains a `kubeconfig` used to access the cluster. +The Kubernetes cluster should be accessible once complete. [Verify](#verify) functionality and then install **important** cluster [addons](#addons). You may cleanup the `bootkube` assets on the node, but you should keep the copy on your laptop. It contains a `kubeconfig` used to access the cluster. ## Verify -[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your laptop. Use the generated kubeconfig to access the Kubernetes cluster. Verify that the cluster is accessible and that the kubelet, apiserver, scheduler, and controller-manager are running as pods. +[Install kubectl](https://coreos.com/kubernetes/docs/latest/configure-kubectl.html) on your laptop. Use the generated kubeconfig to access the Kubernetes cluster. Verify that the cluster is accessible and that the apiserver, scheduler, and controller-manager are running as pods. ```sh $ KUBECONFIG=assets/auth/kubeconfig @@ -128,7 +128,20 @@ kube-system pod-checkpointer-hb960 1/1 Running 0 kube-system pod-checkpointer-hb960-node1.example.com 1/1 Running 0 6m ``` -Try deleting pods to see that the cluster is resilient to failures and machine restarts (Container Linux auto-updates). +## Addons + +Several cluster components run as Kubernetes components. + +### CLUO + +Create the [container-linux-update-operator](https://github.com/coreos/container-linux-update-operator) (i.e. CLUO) `update-operator` deployment and `update-agent` DaemonSet. CLUO coordinates reboots of auto-updating Container Linux nodes so that one node reboots at a time and nodes are drained before reboot. + +``` +kubectl apply -f examples/addons/cluo/update-operator.yaml +kubectl apply -f examples/addons/cluo/update-agent.yaml +``` + +Note that CLUO replaces `locksmithd` reboot coordination. The `update_engine` systemd unit on hosts still performs the Container Linux update check, download, and install to the inactive partition. ## Going further diff --git a/examples/addons/cluo/update-agent.yaml b/examples/addons/cluo/update-agent.yaml new file mode 100644 index 00000000..d0135547 --- /dev/null +++ b/examples/addons/cluo/update-agent.yaml @@ -0,0 +1,55 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: container-linux-update-agent + namespace: kube-system +spec: + template: + metadata: + labels: + app: container-linux-update-agent + container-linux-update.v1.coreos.com/agent-version: v0.2.1 + annotations: + container-linux-update.v1.coreos.com/agent-version: v0.2.1 + spec: + containers: + - name: update-agent + image: quay.io/coreos/container-linux-update-operator:v0.2.1 + command: + - "/bin/update-agent" + volumeMounts: + - mountPath: /var/run/dbus + name: var-run-dbus + - mountPath: /etc/coreos + name: etc-coreos + - mountPath: /usr/share/coreos + name: usr-share-coreos + - mountPath: /etc/os-release + name: etc-os-release + env: + # read by update-agent as the node name to manage reboots for + - name: UPDATE_AGENT_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: var-run-dbus + hostPath: + path: /var/run/dbus + - name: etc-coreos + hostPath: + path: /etc/coreos + - name: usr-share-coreos + hostPath: + path: /usr/share/coreos + - name: etc-os-release + hostPath: + path: /etc/os-release diff --git a/examples/addons/cluo/update-operator.yaml b/examples/addons/cluo/update-operator.yaml new file mode 100644 index 00000000..5aa76f0a --- /dev/null +++ b/examples/addons/cluo/update-operator.yaml @@ -0,0 +1,23 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: container-linux-update-operator + namespace: kube-system +spec: + replicas: 1 + template: + metadata: + labels: + app: container-linux-update-operator + spec: + containers: + - name: update-operator + image: quay.io/coreos/container-linux-update-operator:v0.2.1 + command: + - "/bin/update-operator" + - "--manage-agent=false" + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace diff --git a/examples/groups/bootkube-install/node1.json b/examples/groups/bootkube-install/node1.json index dba26a73..b65e16ab 100644 --- a/examples/groups/bootkube-install/node1.json +++ b/examples/groups/bootkube-install/node1.json @@ -9,11 +9,11 @@ "metadata": { "domain_name": "node1.example.com", "etcd_initial_cluster": "node1=https://node1.example.com:2380", - "etcd_endpoints": "https://node1.example.com:2379", "etcd_name": "node1", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ - "ADD ME" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPQFdwVLr+alsWIgYRz9OdqDhnx9jjuFbkdSdpqq4gd9uZApYlivMDD4UgjFazQpezx8DiNhu9ym7i6LgAcdwi+10hE4L9yoJv9uBgbBxOAd65znqLqF91NtV4mlKP5YfJtR7Ehs+pTB+IIC+o5veDbPn+BYgDMJ2x7Osbn1/gFSDken/yoOFbYbRMGMfVEQYjJzC4r/qCKH0bl/xuVNLxf9FkWSTCcQFKGOndwuGITDkshD4r2Kk8gUddXPxoahBv33/2QH0CY5zbKYjhgN6I6WtwO+O1uJwtNeV1AGhYjurdd60qggNwx+W7623uK3nIXvJd3hzDO8u5oa53/tIL fake-test-key-REMOVE-ME" + ] } } diff --git a/examples/groups/bootkube-install/node2.json b/examples/groups/bootkube-install/node2.json index d365f55b..9f23e40a 100644 --- a/examples/groups/bootkube-install/node2.json +++ b/examples/groups/bootkube-install/node2.json @@ -8,10 +8,9 @@ }, "metadata": { "domain_name": "node2.example.com", - "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ - "ADD ME" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPQFdwVLr+alsWIgYRz9OdqDhnx9jjuFbkdSdpqq4gd9uZApYlivMDD4UgjFazQpezx8DiNhu9ym7i6LgAcdwi+10hE4L9yoJv9uBgbBxOAd65znqLqF91NtV4mlKP5YfJtR7Ehs+pTB+IIC+o5veDbPn+BYgDMJ2x7Osbn1/gFSDken/yoOFbYbRMGMfVEQYjJzC4r/qCKH0bl/xuVNLxf9FkWSTCcQFKGOndwuGITDkshD4r2Kk8gUddXPxoahBv33/2QH0CY5zbKYjhgN6I6WtwO+O1uJwtNeV1AGhYjurdd60qggNwx+W7623uK3nIXvJd3hzDO8u5oa53/tIL fake-test-key-REMOVE-ME" ] } } diff --git a/examples/groups/bootkube-install/node3.json b/examples/groups/bootkube-install/node3.json index a4af27a6..78ece7b6 100644 --- a/examples/groups/bootkube-install/node3.json +++ b/examples/groups/bootkube-install/node3.json @@ -8,10 +8,9 @@ }, "metadata": { "domain_name": "node3.example.com", - "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "ssh_authorized_keys": [ - "ADD ME" + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPQFdwVLr+alsWIgYRz9OdqDhnx9jjuFbkdSdpqq4gd9uZApYlivMDD4UgjFazQpezx8DiNhu9ym7i6LgAcdwi+10hE4L9yoJv9uBgbBxOAd65znqLqF91NtV4mlKP5YfJtR7Ehs+pTB+IIC+o5veDbPn+BYgDMJ2x7Osbn1/gFSDken/yoOFbYbRMGMfVEQYjJzC4r/qCKH0bl/xuVNLxf9FkWSTCcQFKGOndwuGITDkshD4r2Kk8gUddXPxoahBv33/2QH0CY5zbKYjhgN6I6WtwO+O1uJwtNeV1AGhYjurdd60qggNwx+W7623uK3nIXvJd3hzDO8u5oa53/tIL fake-test-key-REMOVE-ME" ] } } diff --git a/examples/groups/bootkube/node1.json b/examples/groups/bootkube/node1.json index c8013370..cdc86c5e 100644 --- a/examples/groups/bootkube/node1.json +++ b/examples/groups/bootkube/node1.json @@ -8,7 +8,6 @@ "metadata": { "domain_name": "node1.example.com", "etcd_initial_cluster": "node1=https://node1.example.com:2380", - "etcd_endpoints": "https://node1.example.com:2379", "etcd_name": "node1", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", diff --git a/examples/groups/bootkube/node2.json b/examples/groups/bootkube/node2.json index 805d9287..70405473 100644 --- a/examples/groups/bootkube/node2.json +++ b/examples/groups/bootkube/node2.json @@ -7,7 +7,6 @@ }, "metadata": { "domain_name": "node2.example.com", - "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", "ssh_authorized_keys": [ diff --git a/examples/groups/bootkube/node3.json b/examples/groups/bootkube/node3.json index 9f8dad22..b81ae98e 100644 --- a/examples/groups/bootkube/node3.json +++ b/examples/groups/bootkube/node3.json @@ -7,7 +7,6 @@ }, "metadata": { "domain_name": "node3.example.com", - "etcd_endpoints": "https://node1.example.com:2379", "k8s_dns_service_ip": "10.3.0.10", "pxe": "true", "ssh_authorized_keys": [ diff --git a/examples/ignition/bootkube-controller.yaml b/examples/ignition/bootkube-controller.yaml index 38901346..185f01e2 100644 --- a/examples/ignition/bootkube-controller.yaml +++ b/examples/ignition/bootkube-controller.yaml @@ -27,15 +27,7 @@ systemd: - name: docker.service enable: true - name: locksmithd.service - dropins: - - name: 40-etcd-lock.conf - contents: | - [Service] - Environment="REBOOT_STRATEGY=etcd-lock" - Environment="LOCKSMITHD_ETCD_CAFILE=/etc/ssl/etcd/etcd-client-ca.crt" - Environment="LOCKSMITHD_ETCD_CERTFILE=/etc/ssl/etcd/etcd-client.crt" - Environment="LOCKSMITHD_ETCD_KEYFILE=/etc/ssl/etcd/etcd-client.key" - Environment="LOCKSMITHD_ENDPOINT={{.etcd_endpoints}}" + mask: true - name: kubelet.path enable: true contents: | diff --git a/examples/ignition/bootkube-worker.yaml b/examples/ignition/bootkube-worker.yaml index 45a8f9d0..151fe617 100644 --- a/examples/ignition/bootkube-worker.yaml +++ b/examples/ignition/bootkube-worker.yaml @@ -4,15 +4,7 @@ systemd: - name: docker.service enable: true - name: locksmithd.service - dropins: - - name: 40-etcd-lock.conf - contents: | - [Service] - Environment="REBOOT_STRATEGY=etcd-lock" - Environment="LOCKSMITHD_ETCD_CAFILE=/etc/ssl/etcd/etcd-client-ca.crt" - Environment="LOCKSMITHD_ETCD_CERTFILE=/etc/ssl/etcd/etcd-client.crt" - Environment="LOCKSMITHD_ETCD_KEYFILE=/etc/ssl/etcd/etcd-client.key" - Environment="LOCKSMITHD_ENDPOINT={{.etcd_endpoints}}" + mask: true - name: kubelet.path enable: true contents: |