From eda78db08ef12c33ee8cd07006171b41947f105d Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sun, 18 Oct 2020 22:51:25 -0700 Subject: [PATCH] Change Flatcar kubelet.service container from rkt to docker * Use docker to run the `kubelet.service` container * Update Kubelet mounts to match Fedora CoreOS * Remove unused `/etc/ssl/certs` mount (see https://github.com/poseidon/typhoon/pull/810) * Remove unused `/usr/share/ca-certificates` mount * Remove `/etc/resolv.conf` mount, Docker default is ok * Change `delete-node.service` to use docker instead of rkt and inline ExecStart, as was done on Fedora CoreOS * Fix permission denied on shutdown `delete-node`, caused by the kubeconfig mount changing with the introduction of node TLS bootstrap Background * podmand, rkt, and runc daemonless container process runners provide advantages over the docker daemon for system containers. Docker requires workarounds for use in systemd units where the ExecStart must tail logs so systemd can monitor the daemonized container. https://github.com/moby/moby/issues/6791 * Why switch then? On Flatcar Linux, podman isn't shipped. rkt works, but isn't developing while container standards continue to move forward. Typhoon has used runc for the Kubelet runner before in Fedora Atomic, but its more low-level. So we're left with Docker, which is less than ideal, but shipped in Flatcar * Flatcar Linux appears to be shifting system components to use docker, which does provide some limited guards against breakages (e.g. Flatcar cannot enable docker live restore) --- CHANGES.md | 6 ++ .../kubernetes/cl/controller.yaml | 59 ++++++------- .../kubernetes/workers/cl/worker.yaml | 85 +++++++------------ .../kubernetes/cl/controller.yaml | 59 ++++++------- .../kubernetes/workers/cl/worker.yaml | 83 +++++++----------- .../kubernetes/cl/controller.yaml | 65 ++++++-------- .../container-linux/kubernetes/cl/worker.yaml | 68 +++++++-------- .../kubernetes/cl/controller.yaml | 59 ++++++------- .../container-linux/kubernetes/cl/worker.yaml | 83 +++++++----------- docs/architecture/operating-systems.md | 2 +- .../kubernetes/cl/controller.yaml | 59 ++++++------- .../kubernetes/workers/cl/worker.yaml | 83 +++++++----------- 12 files changed, 279 insertions(+), 432 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index e3b6a772..6142355d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,12 @@ Notable changes between versions. * Remove `asset_dir` variable (default off in [v1.17.0](https://github.com/poseidon/typhoon/pull/595), deprecated in [v1.18.0](https://github.com/poseidon/typhoon/pull/678)) +### Flatcar Linux + +* Change `kubelet.service` container runner from rkt to docker ([#855](https://github.com/poseidon/typhoon/pull/855)) +* Change `delete-node.service` to be inlined and use docker ([#855](https://github.com/poseidon/typhoon/pull/855)) + * Fix mount to restore permission to delete the local node on shutdown (cloud-only) + ## v1.19.3 * Update Cilium from v1.8.3 to [v1.8.4](https://github.com/cilium/cilium/releases/tag/v1.8.4) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index e49c44ec..3965f560 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -50,9 +50,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -60,39 +62,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -111,7 +98,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 7496643c..c45af1d6 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,9 +23,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -33,39 +35,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -86,21 +76,24 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] WantedBy=multi-user.target - name: delete-node.service - enable: true + enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -117,22 +110,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index e49c44ec..3965f560 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -50,9 +50,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -60,39 +62,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -111,7 +98,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index f96f6bc6..c45af1d6 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,9 +23,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -33,39 +35,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -86,7 +76,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -95,12 +87,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -117,22 +110,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 22ca4493..e30c18d5 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -58,9 +58,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -68,43 +70,26 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume etc-iscsi,kind=host,source=/etc/iscsi \ - --mount volume=etc-iscsi,target=/etc/iscsi \ - --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + -v /etc/iscsi:/etc/iscsi \ + -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -124,7 +109,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index b0215821..a2d7fdaf 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -31,9 +31,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -41,43 +43,29 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume etc-iscsi,kind=host,source=/etc/iscsi \ - --mount volume=etc-iscsi,target=/etc/iscsi \ - --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + -v /etc/iscsi:/etc/iscsi \ + -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -102,7 +90,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 5378c571..e775bfde 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -58,11 +58,13 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -70,39 +72,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -121,7 +108,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 181911ca..601b1599 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -31,11 +31,13 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -43,39 +45,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -93,7 +83,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -102,12 +94,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -122,19 +115,3 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 6cd53e89..174cc938 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -36,7 +36,7 @@ Together, they diversify Typhoon to support a range of container technologies. | kubelet image | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | | control plane images | upstream images | upstream images | | on-host etcd | rkt-fly | podman | -| on-host kubelet | rkt-fly | podman | +| on-host kubelet | docker | podman | | CNI plugins | calico, cilium, flannel | calico, cilium, flannel | | coordinated drain & OS update | [FLUO](https://github.com/kinvolk/flatcar-linux-update-operator) addon | [fleetlock](https://github.com/poseidon/fleetlock) | diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index ad68adba..e791acba 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -50,48 +50,35 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -109,7 +96,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index b0bc2bb0..150122dd 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,48 +23,38 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -84,7 +74,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -93,12 +85,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -115,22 +108,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core