Use mount.Target to specify subdirectory of rootfs mount

- Add Target to mount.Mount.
- Add UnmountMounts to unmount a list of mounts in reverse order.
- Add UnmountRecursive to unmount deepest mount first for a given target, using
moby/sys/mountinfo.

Signed-off-by: Edgar Lee <edgarhinshunlee@gmail.com>

cmd/ctr/commands/snapshots/snapshots.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/tabwriter"
 	"time"
@@ -640,6 +641,6 @@ func printNode(name string, tree *snapshotTree, level int) {
 func printMounts(target string, mounts []mount.Mount) {
 	// FIXME: This is specific to Unix
 	for _, m := range mounts {
-		fmt.Printf("mount -t %s %s %s -o %s\n", m.Type, m.Source, target, strings.Join(m.Options, ","))
+		fmt.Printf("mount -t %s %s %s -o %s\n", m.Type, m.Source, filepath.Join(target, m.Target), strings.Join(m.Options, ","))
 	}
 }

container.go

@@ -258,6 +258,7 @@ func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...N
 			request.Rootfs = append(request.Rootfs, &types.Mount{
 				Type:    m.Type,
 				Source:  m.Source,
+				Target:  m.Target,
 				Options: m.Options,
 			})
 		}
@@ -275,6 +276,7 @@ func (c *container) NewTask(ctx context.Context, ioCreate cio.Creator, opts ...N
 			request.Rootfs = append(request.Rootfs, &types.Mount{
 				Type:    m.Type,
 				Source:  m.Source,
+				Target:  m.Target,
 				Options: m.Options,
 			})
 		}

contrib/snapshotservice/service.go

@@ -205,6 +205,7 @@ func fromMounts(mounts []mount.Mount) []*types.Mount {
 		out[i] = &types.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		}
 	}

diff.go

@@ -128,6 +128,7 @@ func fromMounts(mounts []mount.Mount) []*types.Mount {
 		apiMounts[i] = &types.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		}
 	}

docs/snapshotters/README.md

@@ -24,3 +24,37 @@ Filesystem-specific:
 - `nydus`: [Nydus Snapshotter](https://github.com/containerd/nydus-snapshotter)
 - `overlaybd`: [OverlayBD Snapshotter](https://github.com/containerd/accelerated-container-image)
 - `stargz`: [Stargz Snapshotter](https://github.com/containerd/stargz-snapshotter)
+
+## Mount target
+
+Mounts can optionally specify a target to describe submounts in the container's
+rootfs. For example, if the snapshotter wishes to bind mount to a subdirectory
+ontop of an overlayfs mount, they can return the following mounts:
+
+```json
+[
+    {
+        "type": "overlay",
+        "source": "overlay",
+        "options": [
+            "workdir=...",
+            "upperdir=...",
+            "lowerdir=..."
+        ]
+    },
+    {
+        "type": "bind",
+        "source": "/path/on/host",
+        "target": "/path/inside/container",
+        "options": [
+            "ro",
+            "rbind"
+        ]
+    }
+]
+```
+
+However, the mountpoint `/path/inside/container` needs to exist for the bind
+mount, so one of the previous mounts must be responsible for providing that
+directory in the rootfs. In this case, one of the lower dirs of the overlay has
+that directory to enable the bind mount.

mount/mount.go

@@ -16,6 +16,12 @@
 
 package mount
 
+import (
+	"fmt"
+
+	"github.com/containerd/continuity/fs"
+)
+
 // Mount is the lingua franca of containerd. A mount represents a
 // serialized mount syscall. Components either emit or consume mounts.
 type Mount struct {
@@ -24,12 +30,16 @@ type Mount struct {
 	// Source specifies where to mount from. Depending on the host system, this
 	// can be a source path or device.
 	Source string
+	// Target specifies an optional subdirectory as a mountpoint. It assumes that
+	// the subdirectory exists in a parent mount.
+	Target string
 	// Options contains zero or more fstab-style mount options. Typically,
 	// these are platform specific.
 	Options []string
 }
 
-// All mounts all the provided mounts to the provided target
+// All mounts all the provided mounts to the provided target. If submounts are
+// present, it assumes that parent mounts come before child mounts.
 func All(mounts []Mount, target string) error {
 	for _, m := range mounts {
 		if err := m.Mount(target); err != nil {
@@ -38,3 +48,30 @@ func All(mounts []Mount, target string) error {
 	}
 	return nil
 }
+
+// UnmountMounts unmounts all the mounts under a target in the reverse order of
+// the mounts array provided.
+func UnmountMounts(mounts []Mount, target string, flags int) error {
+	for i := len(mounts) - 1; i >= 0; i-- {
+		mountpoint, err := fs.RootPath(target, mounts[i].Target)
+		if err != nil {
+			return err
+		}
+
+		if err := UnmountAll(mountpoint, flags); err != nil {
+			if i == len(mounts)-1 { // last mount
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// Mount to the provided target path.
+func (m *Mount) Mount(target string) error {
+	target, err := fs.RootPath(target, m.Target)
+	if err != nil {
+		return fmt.Errorf("failed to join path %q with root %q: %w", m.Target, target, err)
+	}
+	return m.mount(target)
+}

mount/mount_freebsd.go

@@ -31,15 +31,12 @@ var (
 	ErrNotImplementOnUnix = errors.New("not implemented under unix")
 )
 
-// Mount to the provided target path.
-func (m *Mount) Mount(target string) error {
-	// The "syscall" and "golang.org/x/sys/unix" packages do not define a Mount
-	// function for FreeBSD, so instead we execute mount(8) and trust it to do
-	// the right thing
-	return m.mountWithHelper(target)
-}
-
-func (m *Mount) mountWithHelper(target string) error {
+// Mount to the provided target.
+//
+// The "syscall" and "golang.org/x/sys/unix" packages do not define a Mount
+// function for FreeBSD, so instead we execute mount(8) and trust it to do
+// the right thing
+func (m *Mount) mount(target string) error {
 	// target: "/foo/target"
 	// command: "mount -o ro -t nullfs /foo/source /foo/merged"
 	// Note: FreeBSD mount(8) is particular about the order of flags and arguments

mount/mount_linux.go

@@ -42,7 +42,7 @@ func init() {
 //
 // If m.Type starts with "fuse." or "fuse3.", "mount.fuse" or "mount.fuse3"
 // helper binary is called.
-func (m *Mount) Mount(target string) (err error) {
+func (m *Mount) mount(target string) (err error) {
 	for _, helperBinary := range allowedHelperBinaries {
 		// helperBinary = "mount.fuse", typePrefix = "fuse."
 		typePrefix := strings.TrimPrefix(helperBinary, "mount.") + "."

mount/mount_linux_test.go

@@ -172,3 +172,67 @@ func TestMountAt(t *testing.T) {
 		t.Fatalf("unexpected working directory: %s", newWD)
 	}
 }
+
+func TestUnmountMounts(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	target, mounts := setupMounts(t)
+	if err := UnmountMounts(mounts, target, 0); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestUnmountRecursive(t *testing.T) {
+	testutil.RequiresRoot(t)
+
+	target, _ := setupMounts(t)
+	if err := UnmountRecursive(target, 0); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func setupMounts(t *testing.T) (target string, mounts []Mount) {
+	dir1 := t.TempDir()
+	dir2 := t.TempDir()
+
+	if err := os.Mkdir(filepath.Join(dir1, "foo"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	mounts = append(mounts, Mount{
+		Type:   "bind",
+		Source: dir1,
+		Options: []string{
+			"ro",
+			"rbind",
+		},
+	})
+
+	if err := os.WriteFile(filepath.Join(dir2, "bar"), []byte("bar"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	mounts = append(mounts, Mount{
+		Type:   "bind",
+		Source: dir2,
+		Target: "foo",
+		Options: []string{
+			"ro",
+			"rbind",
+		},
+	})
+
+	target = t.TempDir()
+	if err := All(mounts, target); err != nil {
+		t.Fatal(err)
+	}
+
+	b, err := os.ReadFile(filepath.Join(target, "foo/bar"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if string(b) != "bar" {
+		t.Fatalf("unexpected file content: %s", b)
+	}
+
+	return target, mounts
+}

mount/mount_unix.go

@@ -1,4 +1,4 @@
-//go:build darwin || openbsd
+//go:build !windows && !darwin && !openbsd
 
 /*
    Copyright The containerd Authors.
@@ -18,24 +18,41 @@
 
 package mount
 
-import "errors"
+import (
+	"sort"
 
-var (
-	// ErrNotImplementOnUnix is returned for methods that are not implemented
-	ErrNotImplementOnUnix = errors.New("not implemented under unix")
+	"github.com/moby/sys/mountinfo"
 )
 
-// Mount is not implemented on this platform
-func (m *Mount) Mount(target string) error {
-	return ErrNotImplementOnUnix
-}
+// UnmountRecursive unmounts the target and all mounts underneath, starting
+// with the deepest mount first.
+func UnmountRecursive(target string, flags int) error {
+	mounts, err := mountinfo.GetMounts(mountinfo.PrefixFilter(target))
+	if err != nil {
+		return err
+	}
 
-// Unmount is not implemented on this platform
-func Unmount(mount string, flags int) error {
-	return ErrNotImplementOnUnix
-}
+	targetSet := make(map[string]struct{})
+	for _, m := range mounts {
+		targetSet[m.Mountpoint] = struct{}{}
+	}
 
-// UnmountAll is not implemented on this platform
-func UnmountAll(mount string, flags int) error {
-	return ErrNotImplementOnUnix
+	var targets []string
+	for m := range targetSet {
+		targets = append(targets, m)
+	}
+
+	// Make the deepest mount be first
+	sort.SliceStable(targets, func(i, j int) bool {
+		return len(targets[i]) > len(targets[j])
+	})
+
+	for i, target := range targets {
+		if err := UnmountAll(target, flags); err != nil {
+			if i == len(targets)-1 { // last mount
+				return err
+			}
+		}
+	}
+	return nil
 }

mount/mount_unsupported.go

@@ -0,0 +1,46 @@
+//go:build darwin || openbsd
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package mount
+
+import "errors"
+
+var (
+	// ErrNotImplementOnUnix is returned for methods that are not implemented
+	ErrNotImplementOnUnix = errors.New("not implemented under unix")
+)
+
+// Mount is not implemented on this platform
+func (m *Mount) mount(target string) error {
+	return ErrNotImplementOnUnix
+}
+
+// Unmount is not implemented on this platform
+func Unmount(mount string, flags int) error {
+	return ErrNotImplementOnUnix
+}
+
+// UnmountAll is not implemented on this platform
+func UnmountAll(mount string, flags int) error {
+	return ErrNotImplementOnUnix
+}
+
+// UnmountRecursive is not implemented on this platform
+func UnmountRecursive(mount string, flags int) error {
+	return ErrNotImplementOnUnix
+}

mount/mount_windows.go

@@ -26,8 +26,8 @@ import (
 	"github.com/Microsoft/hcsshim"
 )
 
-// Mount to the provided target
-func (m *Mount) Mount(target string) error {
+// Mount to the provided target.
+func (m *Mount) mount(target string) error {
 	if m.Type != "windows-layer" {
 		return fmt.Errorf("invalid windows mount type: '%s'", m.Type)
 	}
@@ -58,8 +58,9 @@ func (m *Mount) Mount(target string) error {
 		return fmt.Errorf("failed to get layer mount path for %s: %w", m.Source, err)
 	}
 	mountPath = mountPath + `\`
+
 	if err = os.Symlink(mountPath, target); err != nil {
-		return fmt.Errorf("failed to link mount to taget %s: %w", target, err)
+		return fmt.Errorf("failed to link mount to target %s: %w", target, err)
 	}
 	return nil
 }
@@ -105,3 +106,8 @@ func Unmount(mount string, flags int) error {
 func UnmountAll(mount string, flags int) error {
 	return Unmount(mount, flags)
 }
+
+// UnmountRecursive unmounts from the provided path
+func UnmountRecursive(mount string, flags int) error {
+	return UnmountAll(mount, flags)
+}

mount/temp.go

@@ -49,7 +49,7 @@ func WithTempMount(ctx context.Context, mounts []Mount, f func(root string) erro
 
 	// We should do defer first, if not we will not do Unmount when only a part of Mounts are failed.
 	defer func() {
-		if uerr = UnmountAll(root, 0); uerr != nil {
+		if uerr = UnmountMounts(mounts, root, 0); uerr != nil {
 			uerr = fmt.Errorf("failed to unmount %s: %w", root, uerr)
 			if err == nil {
 				err = uerr

pkg/process/init.go

@@ -307,7 +307,7 @@ func (p *Init) delete(ctx context.Context) error {
 		}
 		p.io.Close()
 	}
-	if err2 := mount.UnmountAll(p.Rootfs, 0); err2 != nil {
+	if err2 := mount.UnmountRecursive(p.Rootfs, 0); err2 != nil {
 		log.G(ctx).WithError(err2).Warn("failed to cleanup rootfs mount")
 		if err == nil {
 			err = fmt.Errorf("failed rootfs umount: %w", err2)

runtime/v1/linux/runtime.go

@@ -251,6 +251,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 		sopts.Rootfs = append(sopts.Rootfs, &types.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		})
 	}

runtime/v1/shim/service.go

@@ -120,9 +120,9 @@ type Service struct {
 
 // Create a new initial process and container with the underlying OCI runtime
 func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *shimapi.CreateTaskResponse, err error) {
-	var mounts []process.Mount
+	var pmounts []process.Mount
 	for _, m := range r.Rootfs {
-		mounts = append(mounts, process.Mount{
+		pmounts = append(pmounts, process.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
 			Target:  m.Target,
@@ -131,7 +131,7 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 	}
 
 	rootfs := ""
-	if len(mounts) > 0 {
+	if len(pmounts) > 0 {
 		rootfs = filepath.Join(r.Bundle, "rootfs")
 		if err := os.Mkdir(rootfs, 0711); err != nil && !os.IsExist(err) {
 			return nil, err
@@ -142,7 +142,7 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 		ID:               r.ID,
 		Bundle:           r.Bundle,
 		Runtime:          r.Runtime,
-		Rootfs:           mounts,
+		Rootfs:           pmounts,
 		Terminal:         r.Terminal,
 		Stdin:            r.Stdin,
 		Stdout:           r.Stdout,
@@ -151,22 +151,24 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *
 		ParentCheckpoint: r.ParentCheckpoint,
 		Options:          r.Options,
 	}
+	var mounts []mount.Mount
+	for _, pm := range pmounts {
+		mounts = append(mounts, mount.Mount{
+			Type:    pm.Type,
+			Source:  pm.Source,
+			Target:  pm.Target,
+			Options: pm.Options,
+		})
+	}
 	defer func() {
 		if err != nil {
-			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+			if err2 := mount.UnmountMounts(mounts, rootfs, 0); err2 != nil {
 				log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount")
 			}
 		}
 	}()
-	for _, rm := range mounts {
-		m := &mount.Mount{
-			Type:    rm.Type,
-			Source:  rm.Source,
-			Options: rm.Options,
-		}
-		if err := m.Mount(rootfs); err != nil {
-			return nil, fmt.Errorf("failed to mount rootfs component %v: %w", m, err)
-		}
+	if err := mount.All(mounts, rootfs); err != nil {
+		return nil, fmt.Errorf("failed to mount rootfs component: %w", err)
 	}
 
 	s.mu.Lock()

runtime/v2/bundle.go

@@ -130,7 +130,7 @@ func (b *Bundle) Delete() error {
 	work, werr := os.Readlink(filepath.Join(b.Path, "work"))
 	rootfs := filepath.Join(b.Path, "rootfs")
 	if runtime.GOOS != "darwin" {
-		if err := mount.UnmountAll(rootfs, 0); err != nil {
+		if err := mount.UnmountRecursive(rootfs, 0); err != nil {
 			return fmt.Errorf("unmount rootfs %s: %w", rootfs, err)
 		}
 	}

runtime/v2/runc/container.go

@@ -59,9 +59,9 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 		}
 	}
 
-	var mounts []process.Mount
+	var pmounts []process.Mount
 	for _, m := range r.Rootfs {
-		mounts = append(mounts, process.Mount{
+		pmounts = append(pmounts, process.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
 			Target:  m.Target,
@@ -70,7 +70,7 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 	}
 
 	rootfs := ""
-	if len(mounts) > 0 {
+	if len(pmounts) > 0 {
 		rootfs = filepath.Join(r.Bundle, "rootfs")
 		if err := os.Mkdir(rootfs, 0711); err != nil && !os.IsExist(err) {
 			return nil, err
@@ -81,7 +81,7 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 		ID:               r.ID,
 		Bundle:           r.Bundle,
 		Runtime:          opts.BinaryName,
-		Rootfs:           mounts,
+		Rootfs:           pmounts,
 		Terminal:         r.Terminal,
 		Stdin:            r.Stdin,
 		Stdout:           r.Stdout,
@@ -98,22 +98,25 @@ func NewContainer(ctx context.Context, platform stdio.Platform, r *task.CreateTa
 	if err := WriteRuntime(r.Bundle, opts.BinaryName); err != nil {
 		return nil, err
 	}
+
+	var mounts []mount.Mount
+	for _, pm := range pmounts {
+		mounts = append(mounts, mount.Mount{
+			Type:    pm.Type,
+			Source:  pm.Source,
+			Target:  pm.Target,
+			Options: pm.Options,
+		})
+	}
 	defer func() {
 		if retErr != nil {
-			if err := mount.UnmountAll(rootfs, 0); err != nil {
+			if err := mount.UnmountMounts(mounts, rootfs, 0); err != nil {
 				logrus.WithError(err).Warn("failed to cleanup rootfs mount")
 			}
 		}
 	}()
-	for _, rm := range mounts {
-		m := &mount.Mount{
-			Type:    rm.Type,
-			Source:  rm.Source,
-			Options: rm.Options,
-		}
-		if err := m.Mount(rootfs); err != nil {
-			return nil, fmt.Errorf("failed to mount rootfs component %v: %w", m, err)
-		}
+	if err := mount.All(mounts, rootfs); err != nil {
+		return nil, fmt.Errorf("failed to mount rootfs component: %w", err)
 	}
 
 	p, err := newInit(

runtime/v2/runc/manager/manager_linux.go

@@ -255,7 +255,7 @@ func (manager) Stop(ctx context.Context, id string) (shim.StopStatus, error) {
 	}); err != nil {
 		log.G(ctx).WithError(err).Warn("failed to remove runc container")
 	}
-	if err := mount.UnmountAll(filepath.Join(path, "rootfs"), 0); err != nil {
+	if err := mount.UnmountRecursive(filepath.Join(path, "rootfs"), 0); err != nil {
 		log.G(ctx).WithError(err).Warn("failed to cleanup rootfs mount")
 	}
 	pid, err := runcC.ReadPidFile(filepath.Join(path, process.InitPidFile))

runtime/v2/runc/v1/service.go

@@ -254,7 +254,7 @@ func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error)
 	}); err != nil {
 		logrus.WithError(err).Warn("failed to remove runc container")
 	}
-	if err := mount.UnmountAll(filepath.Join(path, "rootfs"), 0); err != nil {
+	if err := mount.UnmountRecursive(filepath.Join(path, "rootfs"), 0); err != nil {
 		logrus.WithError(err).Warn("failed to cleanup rootfs mount")
 	}
 

runtime/v2/shim.go

@@ -350,6 +350,7 @@ func (s *shimTask) Create(ctx context.Context, opts runtime.CreateOpts) (runtime
 		request.Rootfs = append(request.Rootfs, &types.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		})
 	}

runtime/v2/shim_load.go

@@ -110,7 +110,7 @@ func (m *ShimManager) loadShims(ctx context.Context) error {
 			container, err := m.containers.Get(ctx, id)
 			if err != nil {
 				log.G(ctx).WithError(err).Errorf("loading container %s", id)
-				if err := mount.UnmountAll(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
+				if err := mount.UnmountRecursive(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
 					log.G(ctx).WithError(err).Errorf("failed to unmount of rootfs %s", id)
 				}
 				bundle.Delete()

services/diff/local.go

@@ -171,6 +171,7 @@ func toMounts(apim []*types.Mount) []mount.Mount {
 		mounts[i] = mount.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		}
 	}

services/snapshots/service.go

@@ -310,6 +310,7 @@ func fromMounts(mounts []mount.Mount) []*types.Mount {
 		out[i] = &types.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		}
 	}

services/tasks/local.go

@@ -215,6 +215,7 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
 		opts.Rootfs = append(opts.Rootfs, mount.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		})
 	}

snapshots/proxy/proxy.go

@@ -226,6 +226,7 @@ func toMounts(mm []*types.Mount) []mount.Mount {
 		mounts[i] = mount.Mount{
 			Type:    m.Type,
 			Source:  m.Source,
+			Target:  m.Target,
 			Options: m.Options,
 		}
 	}