Add WithReadonlyTempMount to create readonly temporary mounts

This is necessary so we can mount snapshots more than once with overlayfs,
otherwise mounts enter an unknown state.

related: https://github.com/moby/buildkit/pull/1100

Signed-off-by: Laura Brehm <laurabrehm@hey.com>
Co-authored-by: Zou Nengren <zouyee1989@gmail.com>

diff/walking/differ.go

@@ -96,7 +96,7 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o
 
 	var ocidesc ocispec.Descriptor
 	if err := mount.WithTempMount(ctx, lower, func(lowerRoot string) error {
-		return mount.WithTempMount(ctx, upper, func(upperRoot string) error {
+		return mount.WithReadonlyTempMount(ctx, upper, func(upperRoot string) error {
 			var newReference bool
 			if config.Reference == "" {
 				newReference = true

mount/mount.go

@@ -18,6 +18,7 @@ package mount
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/containerd/continuity/fs"
 )
@@ -75,3 +76,46 @@ func (m *Mount) Mount(target string) error {
 	}
 	return m.mount(target)
 }
+
+// readonlyMounts modifies the received mount options
+// to make them readonly
+func readonlyMounts(mounts []Mount) []Mount {
+	for i, m := range mounts {
+		if m.Type == "overlay" {
+			mounts[i].Options = readonlyOverlay(m.Options)
+			continue
+		}
+		opts := make([]string, 0, len(m.Options))
+		for _, opt := range m.Options {
+			if opt != "rw" && opt != "ro" { // skip `ro` too so we don't append it twice
+				opts = append(opts, opt)
+			}
+		}
+		opts = append(opts, "ro")
+		mounts[i].Options = opts
+	}
+	return mounts
+}
+
+// readonlyOverlay takes mount options for overlay mounts and makes them readonly by
+// removing workdir and upperdir (and appending the upperdir layer to lowerdir) - see:
+// https://www.kernel.org/doc/html/latest/filesystems/overlayfs.html#multiple-lower-layers
+func readonlyOverlay(opt []string) []string {
+	out := make([]string, 0, len(opt))
+	upper := ""
+	for _, o := range opt {
+		if strings.HasPrefix(o, "upperdir=") {
+			upper = strings.TrimPrefix(o, "upperdir=")
+		} else if !strings.HasPrefix(o, "workdir=") {
+			out = append(out, o)
+		}
+	}
+	if upper != "" {
+		for i, o := range out {
+			if strings.HasPrefix(o, "lowerdir=") {
+				out[i] = "lowerdir=" + upper + ":" + strings.TrimPrefix(o, "lowerdir=")
+			}
+		}
+	}
+	return out
+}

mount/mount_test.go

@@ -0,0 +1,150 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package mount
+
+import (
+	"reflect"
+	"testing"
+
+	// required for `-test.root` flag not to fail
+	_ "github.com/containerd/continuity/testutil"
+)
+
+func TestReadonlyMounts(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		input    []Mount
+		expected []Mount
+	}{
+		{
+			desc:     "empty slice",
+			input:    []Mount{},
+			expected: []Mount{},
+		},
+		{
+			desc: "removes `upperdir` and `workdir` from overlay mounts, appends upper layer to lower",
+			input: []Mount{
+				{
+					Type:   "overlay",
+					Source: "overlay",
+					Options: []string{
+						"index=off",
+						"workdir=/path/to/snapshots/4/work",
+						"upperdir=/path/to/snapshots/4/fs",
+						"lowerdir=/path/to/snapshots/1/fs",
+					},
+				},
+				{
+					Type:   "overlay",
+					Source: "overlay",
+					Options: []string{
+						"index=on",
+						"lowerdir=/another/path/to/snapshots/2/fs",
+					},
+				},
+			},
+			expected: []Mount{
+				{
+					Type:   "overlay",
+					Source: "overlay",
+					Options: []string{
+						"index=off",
+						"lowerdir=/path/to/snapshots/4/fs:/path/to/snapshots/1/fs",
+					},
+				},
+				{
+					Type:   "overlay",
+					Source: "overlay",
+					Options: []string{
+						"index=on",
+						"lowerdir=/another/path/to/snapshots/2/fs",
+					},
+				},
+			},
+		},
+		{
+			desc: "removes `rw` and appends `ro` (once) to other mount types",
+			input: []Mount{
+				{
+					Type:   "mount-without-rw",
+					Source: "",
+					Options: []string{
+						"index=off",
+						"workdir=/path/to/other/snapshots/work",
+						"upperdir=/path/to/other/snapshots/2",
+						"lowerdir=/path/to/other/snapshots/1",
+					},
+				},
+				{
+					Type:   "mount-with-rw",
+					Source: "",
+					Options: []string{
+						"an-option=a-value",
+						"another_opt=/another/value",
+						"rw",
+					},
+				},
+				{
+					Type:   "mount-with-ro",
+					Source: "",
+					Options: []string{
+						"an-option=a-value",
+						"another_opt=/another/value",
+						"ro",
+					},
+				},
+			},
+			expected: []Mount{
+				{
+					Type:   "mount-without-rw",
+					Source: "",
+					Options: []string{
+						"index=off",
+						"workdir=/path/to/other/snapshots/work",
+						"upperdir=/path/to/other/snapshots/2",
+						"lowerdir=/path/to/other/snapshots/1",
+						"ro",
+					},
+				},
+				{
+					Type:   "mount-with-rw",
+					Source: "",
+					Options: []string{
+						"an-option=a-value",
+						"another_opt=/another/value",
+						"ro",
+					},
+				},
+				{
+					Type:   "mount-with-ro",
+					Source: "",
+					Options: []string{
+						"an-option=a-value",
+						"another_opt=/another/value",
+						"ro",
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		if !reflect.DeepEqual(readonlyMounts(tc.input), tc.expected) {
+			t.Fatalf("incorrectly modified mounts: %s", tc.desc)
+		}
+	}
+}

mount/temp.go

@@ -67,6 +67,13 @@ func WithTempMount(ctx context.Context, mounts []Mount, f func(root string) erro
 	return nil
 }
 
+// WithReadonlyTempMount mounts the provided mounts to a temp dir as readonly,
+// and pass the temp dir to f. The mounts are valid during the call to the f.
+// Finally we will unmount and remove the temp dir regardless of the result of f.
+func WithReadonlyTempMount(ctx context.Context, mounts []Mount, f func(root string) error) (err error) {
+	return WithTempMount(ctx, readonlyMounts(mounts), f)
+}
+
 func getTempDir() string {
 	if xdg := os.Getenv("XDG_RUNTIME_DIR"); xdg != "" {
 		return xdg