fix: prevent EBUSY when unmounting system disk

Reading /proc/mounts while simultaneously unmounting mountpoints
prevents unmounting all submounts under /var. This is due to the fact
that /proc/mounts will change as we perform unmounts, and that causes a
read of the file to become inaccurate. We now read /proc/mounts into
memory to get a snapshot of all submounts under /var, and then we
proceed with unmounting them.

This also adds some additional logging that I found to be useful while
debugging this. It also adds logic to skip of DaemonSet managed pods.

Signed-off-by: Andrew Rynhard <andrew@andrewrynhard.com>

internal/app/machined/internal/phase/kubernetes/tasks.go

@@ -6,6 +6,7 @@ package kubernetes
 
 import (
 	"context"
+	"log"
 	"syscall"
 
 	"github.com/containerd/containerd"
@@ -59,6 +60,7 @@ func (task *KillKubernetesTasks) standard() (err error) {
 
 	for _, task := range response.Tasks {
 		task := task // https://golang.org/doc/faq#closures_and_goroutines
+		log.Printf("killing task %s", task.ID)
 		g.Go(func() error {
 			if _, err = s.Kill(ctx, &tasks.KillRequest{ContainerID: task.ID, Signal: uint32(syscall.SIGTERM), All: true}); err != nil {
 				return errors.Wrap(err, "error killing task")

internal/app/machined/internal/phase/rootfs/unmount_pod_mounts.go

@@ -6,7 +6,9 @@ package rootfs
 
 import (
 	"bufio"
-	"os"
+	"bytes"
+	"io/ioutil"
+	"log"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -37,14 +39,14 @@ func (task *UnmountPodMounts) RuntimeFunc(mode runtime.Mode) phase.RuntimeFunc {
 }
 
 func (task *UnmountPodMounts) standard(platform platform.Platform, data *userdata.UserData) (err error) {
-	file, err := os.Open("/proc/mounts")
-	if err != nil {
+	var b []byte
+	if b, err = ioutil.ReadFile("/proc/self/mounts"); err != nil {
 		return err
 	}
-	// nolint: errcheck
-	defer file.Close()
 
-	scanner := bufio.NewScanner(file)
+	r := bytes.NewReader(b)
+
+	scanner := bufio.NewScanner(r)
 	for scanner.Scan() {
 		fields := strings.Fields(scanner.Text())
 
@@ -54,11 +56,15 @@ func (task *UnmountPodMounts) standard(platform platform.Platform, data *userdat
 
 		mountpoint := fields[1]
 		if strings.HasPrefix(mountpoint, constants.EphemeralMountPoint+"/") {
-			if err := unix.Unmount(mountpoint, 0); err != nil {
-				return errors.Errorf("error creating overlay mount to %s: %v", mountpoint, err)
+			log.Printf("unmounting %s\n", mountpoint)
+			if err = unix.Unmount(mountpoint, 0); err != nil {
+				return errors.Errorf("error unmounting %s: %v", mountpoint, err)
 			}
 		}
 	}
+	if err = scanner.Err(); err != nil {
+		return err
+	}
 
 	return nil
 }

pkg/kubernetes/kubernetes.go

@@ -104,6 +104,12 @@ func (h *Helper) Drain(node string) error {
 	for _, pod := range pods.Items {
 		go func(p corev1.Pod) {
 			defer wg.Done()
+			for _, ref := range p.ObjectMeta.OwnerReferences {
+				if ref.Kind == "DaemonSet" {
+					log.Printf("skipping DaemonSet pod %s\n", p.GetName())
+					return
+				}
+			}
 			if err := h.evict(p, int64(60)); err != nil {
 				log.Printf("WARNING: failed to evict pod: %v", err)
 			}