diff --git a/CHANGES.md b/CHANGES.md index e3b6a772..6142355d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,12 @@ Notable changes between versions. * Remove `asset_dir` variable (default off in [v1.17.0](https://github.com/poseidon/typhoon/pull/595), deprecated in [v1.18.0](https://github.com/poseidon/typhoon/pull/678)) +### Flatcar Linux + +* Change `kubelet.service` container runner from rkt to docker ([#855](https://github.com/poseidon/typhoon/pull/855)) +* Change `delete-node.service` to be inlined and use docker ([#855](https://github.com/poseidon/typhoon/pull/855)) + * Fix mount to restore permission to delete the local node on shutdown (cloud-only) + ## v1.19.3 * Update Cilium from v1.8.3 to [v1.8.4](https://github.com/cilium/cilium/releases/tag/v1.8.4) diff --git a/aws/container-linux/kubernetes/cl/controller.yaml b/aws/container-linux/kubernetes/cl/controller.yaml index e49c44ec..3965f560 100644 --- a/aws/container-linux/kubernetes/cl/controller.yaml +++ b/aws/container-linux/kubernetes/cl/controller.yaml @@ -50,9 +50,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -60,39 +62,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -111,7 +98,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/aws/container-linux/kubernetes/workers/cl/worker.yaml b/aws/container-linux/kubernetes/workers/cl/worker.yaml index 7496643c..c45af1d6 100644 --- a/aws/container-linux/kubernetes/workers/cl/worker.yaml +++ b/aws/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,9 +23,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -33,39 +35,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -86,21 +76,24 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] WantedBy=multi-user.target - name: delete-node.service - enable: true + enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -117,22 +110,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core diff --git a/azure/container-linux/kubernetes/cl/controller.yaml b/azure/container-linux/kubernetes/cl/controller.yaml index e49c44ec..3965f560 100644 --- a/azure/container-linux/kubernetes/cl/controller.yaml +++ b/azure/container-linux/kubernetes/cl/controller.yaml @@ -50,9 +50,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -60,39 +62,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -111,7 +98,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/azure/container-linux/kubernetes/workers/cl/worker.yaml b/azure/container-linux/kubernetes/workers/cl/worker.yaml index f96f6bc6..c45af1d6 100644 --- a/azure/container-linux/kubernetes/workers/cl/worker.yaml +++ b/azure/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,9 +23,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -33,39 +35,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -86,7 +76,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -95,12 +87,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -117,22 +110,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname | tr '[:upper:]' '[:lower:]') passwd: users: - name: core diff --git a/bare-metal/container-linux/kubernetes/cl/controller.yaml b/bare-metal/container-linux/kubernetes/cl/controller.yaml index 22ca4493..e30c18d5 100644 --- a/bare-metal/container-linux/kubernetes/cl/controller.yaml +++ b/bare-metal/container-linux/kubernetes/cl/controller.yaml @@ -58,9 +58,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -68,43 +70,26 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume etc-iscsi,kind=host,source=/etc/iscsi \ - --mount volume=etc-iscsi,target=/etc/iscsi \ - --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + -v /etc/iscsi:/etc/iscsi \ + -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -124,7 +109,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/bare-metal/container-linux/kubernetes/cl/worker.yaml b/bare-metal/container-linux/kubernetes/cl/worker.yaml index b0215821..a2d7fdaf 100644 --- a/bare-metal/container-linux/kubernetes/cl/worker.yaml +++ b/bare-metal/container-linux/kubernetes/cl/worker.yaml @@ -31,9 +31,11 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Environment=KUBELET_CGROUP_DRIVER=${cgroup_driver} ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -41,43 +43,29 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - --volume etc-iscsi,kind=host,source=/etc/iscsi \ - --mount volume=etc-iscsi,target=/etc/iscsi \ - --volume usr-sbin-iscsiadm,kind=host,source=/usr/sbin/iscsiadm \ - --mount volume=usr-sbin-iscsiadm,target=/sbin/iscsiadm \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + -v /etc/iscsi:/etc/iscsi \ + -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -102,7 +90,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] diff --git a/digital-ocean/container-linux/kubernetes/cl/controller.yaml b/digital-ocean/container-linux/kubernetes/cl/controller.yaml index 5378c571..e775bfde 100644 --- a/digital-ocean/container-linux/kubernetes/cl/controller.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/controller.yaml @@ -58,11 +58,13 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -70,39 +72,24 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -121,7 +108,9 @@ systemd: --register-with-taints=node-role.kubernetes.io/controller=:NoSchedule \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/digital-ocean/container-linux/kubernetes/cl/worker.yaml b/digital-ocean/container-linux/kubernetes/cl/worker.yaml index 181911ca..601b1599 100644 --- a/digital-ocean/container-linux/kubernetes/cl/worker.yaml +++ b/digital-ocean/container-linux/kubernetes/cl/worker.yaml @@ -31,11 +31,13 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Requires=coreos-metadata.service After=coreos-metadata.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 EnvironmentFile=/run/metadata/coreos ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests @@ -43,39 +45,27 @@ systemd: ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -93,7 +83,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -102,12 +94,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -122,19 +115,3 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) diff --git a/docs/architecture/operating-systems.md b/docs/architecture/operating-systems.md index 6cd53e89..174cc938 100644 --- a/docs/architecture/operating-systems.md +++ b/docs/architecture/operating-systems.md @@ -36,7 +36,7 @@ Together, they diversify Typhoon to support a range of container technologies. | kubelet image | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | kubelet [image](https://github.com/poseidon/kubelet) with upstream binary | | control plane images | upstream images | upstream images | | on-host etcd | rkt-fly | podman | -| on-host kubelet | rkt-fly | podman | +| on-host kubelet | docker | podman | | CNI plugins | calico, cilium, flannel | calico, cilium, flannel | | coordinated drain & OS update | [FLUO](https://github.com/kinvolk/flatcar-linux-update-operator) addon | [fleetlock](https://github.com/poseidon/fleetlock) | diff --git a/google-cloud/container-linux/kubernetes/cl/controller.yaml b/google-cloud/container-linux/kubernetes/cl/controller.yaml index ad68adba..e791acba 100644 --- a/google-cloud/container-linux/kubernetes/cl/controller.yaml +++ b/google-cloud/container-linux/kubernetes/cl/controller.yaml @@ -50,48 +50,35 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -109,7 +96,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=10 [Install] diff --git a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml index b0bc2bb0..150122dd 100644 --- a/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml +++ b/google-cloud/container-linux/kubernetes/workers/cl/worker.yaml @@ -23,48 +23,38 @@ systemd: contents: | [Unit] Description=Kubelet + Requires=docker.service + After=docker.service Wants=rpc-statd.service [Service] - Environment=KUBELET_IMAGE=docker://quay.io/poseidon/kubelet:v1.19.3 + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 ExecStartPre=/bin/mkdir -p /etc/kubernetes/cni/net.d ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests ExecStartPre=/bin/mkdir -p /opt/cni/bin ExecStartPre=/bin/mkdir -p /var/lib/calico ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt" - ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid - ExecStart=/usr/bin/rkt run \ - --uuid-file-save=/var/cache/kubelet-pod.uuid \ - --stage1-from-dir=stage1-fly.aci \ - --hosts-entry host \ - --insecure-options=image \ - --volume etc-kubernetes,kind=host,source=/etc/kubernetes,readOnly=true \ - --mount volume=etc-kubernetes,target=/etc/kubernetes \ - --volume etc-machine-id,kind=host,source=/etc/machine-id,readOnly=true \ - --mount volume=etc-machine-id,target=/etc/machine-id \ - --volume etc-os-release,kind=host,source=/usr/lib/os-release,readOnly=true \ - --mount volume=etc-os-release,target=/etc/os-release \ - --volume=etc-resolv,kind=host,source=/etc/resolv.conf,readOnly=true \ - --mount volume=etc-resolv,target=/etc/resolv.conf \ - --volume etc-ssl-certs,kind=host,source=/etc/ssl/certs,readOnly=true \ - --mount volume=etc-ssl-certs,target=/etc/ssl/certs \ - --volume lib-modules,kind=host,source=/lib/modules,readOnly=true \ - --mount volume=lib-modules,target=/lib/modules \ - --volume run,kind=host,source=/run \ - --mount volume=run,target=/run \ - --volume usr-share-certs,kind=host,source=/usr/share/ca-certificates,readOnly=true \ - --mount volume=usr-share-certs,target=/usr/share/ca-certificates \ - --volume var-lib-calico,kind=host,source=/var/lib/calico,readOnly=true \ - --mount volume=var-lib-calico,target=/var/lib/calico \ - --volume var-lib-docker,kind=host,source=/var/lib/docker \ - --mount volume=var-lib-docker,target=/var/lib/docker \ - --volume var-lib-kubelet,kind=host,source=/var/lib/kubelet,recursive=true \ - --mount volume=var-lib-kubelet,target=/var/lib/kubelet \ - --volume var-log,kind=host,source=/var/log \ - --mount volume=var-log,target=/var/log \ - --volume opt-cni-bin,kind=host,source=/opt/cni/bin \ - --mount volume=opt-cni-bin,target=/opt/cni/bin \ - $${KUBELET_IMAGE} -- \ + # Podman, rkt, or runc run container processes, whereas docker run + # is a client to a daemon and requires workarounds to use within a + # systemd unit. https://github.com/moby/moby/issues/6791 + ExecStartPre=/usr/bin/docker run -d \ + --name kubelet \ + --privileged \ + --pid host \ + --network host \ + -v /etc/kubernetes:/etc/kubernetes:ro \ + -v /etc/machine-id:/etc/machine-id:ro \ + -v /usr/lib/os-release:/etc/os-release:ro \ + -v /lib/modules:/lib/modules:ro \ + -v /run:/run \ + -v /sys/fs/cgroup:/sys/fs/cgroup:ro \ + -v /sys/fs/cgroup/systemd:/sys/fs/cgroup/systemd \ + -v /var/lib/calico:/var/lib/calico:ro \ + -v /var/lib/docker:/var/lib/docker \ + -v /var/lib/kubelet:/var/lib/kubelet:rshared \ + -v /var/log:/var/log \ + -v /opt/cni/bin:/opt/cni/bin \ + $${KUBELET_IMAGE} \ --anonymous-auth=false \ --authentication-token-webhook \ --authorization-mode=Webhook \ @@ -84,7 +74,9 @@ systemd: --read-only-port=0 \ --rotate-certificates \ --volume-plugin-dir=/var/lib/kubelet/volumeplugins - ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid + ExecStart=docker logs -f kubelet + ExecStop=docker stop kubelet + ExecStopPost=docker rm kubelet Restart=always RestartSec=5 [Install] @@ -93,12 +85,13 @@ systemd: enabled: true contents: | [Unit] - Description=Waiting to delete Kubernetes node on shutdown + Description=Delete Kubernetes node on shutdown [Service] + Environment=KUBELET_IMAGE=quay.io/poseidon/kubelet:v1.19.3 Type=oneshot RemainAfterExit=true ExecStart=/bin/true - ExecStop=/etc/kubernetes/delete-node + ExecStop=/bin/bash -c '/usr/bin/docker run -v /var/lib/kubelet:/var/lib/kubelet:ro --entrypoint /usr/local/bin/kubectl $${KUBELET_IMAGE} --kubeconfig=/var/lib/kubelet/kubeconfig delete node $HOSTNAME' [Install] WantedBy=multi-user.target storage: @@ -115,22 +108,6 @@ storage: contents: inline: | fs.inotify.max_user_watches=16184 - - path: /etc/kubernetes/delete-node - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - set -e - exec /usr/bin/rkt run \ - --trust-keys-from-https \ - --volume config,kind=host,source=/etc/kubernetes \ - --mount volume=config,target=/etc/kubernetes \ - --insecure-options=image \ - docker://quay.io/poseidon/kubelet:v1.19.3 \ - --net=host \ - --dns=host \ - --exec=/usr/local/bin/kubectl -- --kubeconfig=/etc/kubernetes/kubeconfig delete node $(hostname) passwd: users: - name: core