Use local "ensureRemoveAll" instead of docker/pkg/system

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

pkg/server/helpers_unix.go

@@ -19,17 +19,24 @@ limitations under the License.
 package server
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
+	"syscall"
+	"time"
 
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
 	runcapparmor "github.com/opencontainers/runc/libcontainer/apparmor"
 	runcseccomp "github.com/opencontainers/runc/libcontainer/seccomp"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
+	"golang.org/x/sys/unix"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
@@ -141,3 +148,106 @@ func (c *criService) seccompEnabled() bool {
 func openLogFile(path string) (*os.File, error) {
 	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
 }
+
+// unmountRecursive unmounts the target and all mounts underneath, starting with
+// the deepest mount first.
+func unmountRecursive(ctx context.Context, target string) error {
+	mounts, err := mount.Self()
+	if err != nil {
+		return err
+	}
+
+	var toUnmount []string
+	for _, m := range mounts {
+		p, err := filepath.Rel(target, m.Mountpoint)
+		if err != nil {
+			return err
+		}
+		if !strings.HasPrefix(p, "..") {
+			toUnmount = append(toUnmount, m.Mountpoint)
+		}
+	}
+
+	// Make the deepest mount be first
+	sort.Slice(toUnmount, func(i, j int) bool {
+		return len(toUnmount[i]) > len(toUnmount[j])
+	})
+
+	for i, mountPath := range toUnmount {
+		if err := mount.UnmountAll(mountPath, unix.MNT_DETACH); err != nil {
+			if i == len(toUnmount)-1 { // last mount
+				return err
+			}
+			// This is some submount, we can ignore this error for now, the final unmount will fail if this is a real problem
+			log.G(ctx).WithError(err).Debugf("failed to unmount submount %s", mountPath)
+		}
+	}
+	return nil
+}
+
+// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
+// often be remedied.
+// Only use `ensureRemoveAll` if you really want to make every effort to remove
+// a directory.
+//
+// Because of the way `os.Remove` (and by extension `os.RemoveAll`) works, there
+// can be a race between reading directory entries and then actually attempting
+// to remove everything in the directory.
+// These types of errors do not need to be returned since it's ok for the dir to
+// be gone we can just retry the remove operation.
+//
+// This should not return a `os.ErrNotExist` kind of error under any circumstances
+func ensureRemoveAll(ctx context.Context, dir string) error {
+	notExistErr := make(map[string]bool)
+
+	// track retries
+	exitOnErr := make(map[string]int)
+	maxRetry := 50
+
+	// Attempt to unmount anything beneath this dir first.
+	if err := unmountRecursive(ctx, dir); err != nil {
+		log.G(ctx).WithError(err).Debugf("failed to do initial unmount of %s", dir)
+	}
+
+	for {
+		err := os.RemoveAll(dir)
+		if err == nil {
+			return nil
+		}
+
+		pe, ok := err.(*os.PathError)
+		if !ok {
+			return err
+		}
+
+		if os.IsNotExist(err) {
+			if notExistErr[pe.Path] {
+				return err
+			}
+			notExistErr[pe.Path] = true
+
+			// There is a race where some subdir can be removed but after the
+			// parent dir entries have been read.
+			// So the path could be from `os.Remove(subdir)`
+			// If the reported non-existent path is not the passed in `dir` we
+			// should just retry, but otherwise return with no error.
+			if pe.Path == dir {
+				return nil
+			}
+			continue
+		}
+
+		if pe.Err != syscall.EBUSY {
+			return err
+		}
+		if e := mount.Unmount(pe.Path, unix.MNT_DETACH); e != nil {
+			return errors.Wrapf(e, "error while removing %s", dir)
+		}
+
+		if exitOnErr[pe.Path] == maxRetry {
+			return err
+		}
+		exitOnErr[pe.Path]++
+		time.Sleep(100 * time.Millisecond)
+	}
+}