Merge pull request #5076 from AkihiroSuda/ovl-k511

overlay: support "userxattr" option (kernel 5.11)

snapshots/overlay/check.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/sys"
 	"github.com/containerd/continuity/fs"
 	"github.com/pkg/errors"
 )
@@ -86,3 +87,84 @@ func Supported(root string) error {
 	}
 	return supportsMultipleLowerDir(root)
 }
+
+// NeedsUserXAttr returns whether overlayfs should be mounted with the "userxattr" mount option.
+//
+// The "userxattr" option is needed for mounting overlayfs inside a user namespace with kernel >= 5.11.
+//
+// The "userxattr" option is NOT needed for the initial user namespace (aka "the host").
+//
+// Also, Ubuntu (since circa 2015) and Debian (since 10) with kernel < 5.11 can mount
+// the overlayfs in a user namespace without the "userxattr" option.
+//
+// The corresponding kernel commit: https://github.com/torvalds/linux/commit/2d2f2d7322ff43e0fe92bf8cccdc0b09449bf2e1
+// > ovl: user xattr
+// >
+// > Optionally allow using "user.overlay." namespace instead of "trusted.overlay."
+// > ...
+// > Disable redirect_dir and metacopy options, because these would allow privilege escalation through direct manipulation of the
+// > "user.overlay.redirect" or "user.overlay.metacopy" xattrs.
+// > ...
+//
+// The "userxattr" support is not exposed in "/sys/module/overlay/parameters".
+func NeedsUserXAttr(d string) (bool, error) {
+	if !sys.RunningInUserNS() {
+		// we are the real root (i.e., the root in the initial user NS),
+		// so we do never need "userxattr" opt.
+		return false, nil
+	}
+
+	// TODO: add fast path for kernel >= 5.11 .
+	//
+	// Keep in mind that distro vendors might be going to backport the patch to older kernels.
+	// So we can't completely remove the check.
+
+	tdRoot := filepath.Join(d, "userxattr-check")
+	if err := os.RemoveAll(tdRoot); err != nil {
+		log.L.WithError(err).Warnf("Failed to remove check directory %v", tdRoot)
+	}
+
+	if err := os.MkdirAll(tdRoot, 0700); err != nil {
+		return false, err
+	}
+
+	defer func() {
+		if err := os.RemoveAll(tdRoot); err != nil {
+			log.L.WithError(err).Warnf("Failed to remove check directory %v", tdRoot)
+		}
+	}()
+
+	td, err := ioutil.TempDir(tdRoot, "")
+	if err != nil {
+		return false, err
+	}
+
+	for _, dir := range []string{"lower1", "lower2", "upper", "work", "merged"} {
+		if err := os.Mkdir(filepath.Join(td, dir), 0755); err != nil {
+			return false, err
+		}
+	}
+
+	opts := []string{
+		fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", filepath.Join(td, "lower2"), filepath.Join(td, "lower1"), filepath.Join(td, "upper"), filepath.Join(td, "work")),
+		"userxattr",
+	}
+
+	m := mount.Mount{
+		Type:    "overlay",
+		Source:  "overlay",
+		Options: opts,
+	}
+
+	dest := filepath.Join(td, "merged")
+	if err := m.Mount(dest); err != nil {
+		// Probably the host is running Ubuntu/Debian kernel (< 5.11) with the userns patch but without the userxattr patch.
+		// Return false without error.
+		log.L.WithError(err).Debugf("cannot mount overlay with \"userxattr\", probably the kernel does not support userxattr")
+		return false, nil
+	}
+	if err := mount.UnmountAll(dest, 0); err != nil {
+		log.L.WithError(err).Warnf("Failed to unmount check directory %v", dest)
+	}
+	return true, nil
+}

snapshots/overlay/overlay.go

@@ -33,6 +33,7 @@ import (
 	"github.com/containerd/containerd/snapshots/storage"
 	"github.com/containerd/continuity/fs"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 // SnapshotterConfig is used to configure the overlay snapshotter instance
@@ -57,6 +58,7 @@ type snapshotter struct {
 	ms          *storage.MetaStore
 	asyncRemove bool
 	indexOff    bool
+	userxattr   bool // whether to enable "userxattr" mount option
 }
 
 // NewSnapshotter returns a Snapshotter which uses overlayfs. The overlayfs
@@ -95,11 +97,18 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 		indexOff = true
 	}
 
+	// figure out whether "userxattr" option is recognized by the kernel && needed
+	userxattr, err := NeedsUserXAttr(root)
+	if err != nil {
+		logrus.WithError(err).Warnf("cannot detect whether \"userxattr\" option needs to be used, assuming to be %v", userxattr)
+	}
+
 	return &snapshotter{
 		root:        root,
 		ms:          ms,
 		asyncRemove: config.asyncRemove,
 		indexOff:    indexOff,
+		userxattr:   userxattr,
 	}, nil
 }
 
@@ -464,6 +473,10 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount {
 		options = append(options, "index=off")
 	}
 
+	if o.userxattr {
+		options = append(options, "userxattr")
+	}
+
 	if s.Kind == snapshots.KindActive {
 		options = append(options,
 			fmt.Sprintf("workdir=%s", o.workPath(s.ID)),

snapshots/overlay/overlay_test.go

@@ -173,12 +173,21 @@ func testOverlayOverlayMount(t *testing.T, newSnapshotter testsuite.SnapshotterF
 		upper = "upperdir=" + filepath.Join(bp, "fs")
 		lower = "lowerdir=" + getParents(ctx, o, root, "/tmp/layer2")[0]
 	)
-	for i, v := range []string{
+
+	expected := []string{
 		"index=off",
+	}
+	if userxattr, err := NeedsUserXAttr(root); err != nil {
+		t.Fatal(err)
+	} else if userxattr {
+		expected = append(expected, "userxattr")
+	}
+	expected = append(expected, []string{
 		work,
 		upper,
 		lower,
-	} {
+	}...)
+	for i, v := range expected {
 		if m.Options[i] != v {
 			t.Errorf("expected %q but received %q", v, m.Options[i])
 		}
@@ -335,12 +344,26 @@ func testOverlayView(t *testing.T, newSnapshotter testsuite.SnapshotterFunc) {
 	if m.Source != "overlay" {
 		t.Errorf("mount source should be overlay but received %q", m.Source)
 	}
-	if len(m.Options) != 2 {
-		t.Errorf("expected 1 additional mount option but got %d", len(m.Options))
+
+	expectedOptions := 2
+	userxattr, err := NeedsUserXAttr(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if userxattr {
+		expectedOptions++
+	}
+
+	if len(m.Options) != expectedOptions {
+		t.Errorf("expected %d additional mount option but got %d", expectedOptions, len(m.Options))
 	}
 	lowers := getParents(ctx, o, root, "/tmp/view2")
 	expected = fmt.Sprintf("lowerdir=%s:%s", lowers[0], lowers[1])
-	if m.Options[1] != expected {
-		t.Errorf("expected option %q but received %q", expected, m.Options[0])
+	optIdx := 1
+	if userxattr {
+		optIdx++
+	}
+	if m.Options[optIdx] != expected {
+		t.Errorf("expected option %q but received %q", expected, m.Options[optIdx])
 	}
 }