Move snapshots to core/snapshots

Signed-off-by: Derek McGowan <derek@mcg.dev>

core/snapshots/testsuite/helpers.go

@@ -0,0 +1,170 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testsuite
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/containerd/v2/pkg/randutil"
+	"github.com/containerd/continuity/fs/fstest"
+)
+
+const umountflags int = 0
+
+func applyToMounts(m []mount.Mount, work string, a fstest.Applier) (err error) {
+	td, err := os.MkdirTemp(work, "prepare")
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer os.RemoveAll(td)
+
+	if err := mount.All(m, td); err != nil {
+		return fmt.Errorf("failed to mount: %w", err)
+	}
+	defer func() {
+		if err1 := mount.UnmountAll(td, umountflags); err1 != nil && err == nil {
+			err = fmt.Errorf("failed to unmount: %w", err1)
+		}
+	}()
+
+	return a.Apply(td)
+}
+
+// createSnapshot creates a new snapshot in the snapshotter
+// given an applier to run on top of the given parent.
+func createSnapshot(ctx context.Context, sn snapshots.Snapshotter, parent, work string, a fstest.Applier) (string, error) {
+	n := fmt.Sprintf("%p-%d", a, randutil.Int())
+	prepare := fmt.Sprintf("%s-prepare", n)
+
+	m, err := sn.Prepare(ctx, prepare, parent, opt)
+	if err != nil {
+		return "", fmt.Errorf("failed to prepare snapshot: %w", err)
+	}
+
+	if err := applyToMounts(m, work, a); err != nil {
+		return "", fmt.Errorf("failed to apply: %w", err)
+	}
+
+	if err := sn.Commit(ctx, n, prepare, opt); err != nil {
+		return "", fmt.Errorf("failed to commit: %w", err)
+	}
+
+	return n, nil
+}
+
+func checkSnapshot(ctx context.Context, sn snapshots.Snapshotter, work, name, check string) (err error) {
+	td, err := os.MkdirTemp(work, "check")
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer func() {
+		if err1 := os.RemoveAll(td); err1 != nil && err == nil {
+			err = fmt.Errorf("failed to remove temporary directory %s: %w", td, err1)
+		}
+	}()
+
+	view := fmt.Sprintf("%s-view", name)
+	m, err := sn.View(ctx, view, name, opt)
+	if err != nil {
+		return fmt.Errorf("failed to create view: %w", err)
+	}
+	defer func() {
+		if err1 := sn.Remove(ctx, view); err1 != nil && err == nil {
+			err = fmt.Errorf("failed to remove view: %w", err1)
+		}
+	}()
+
+	if err := mount.All(m, td); err != nil {
+		return fmt.Errorf("failed to mount: %w", err)
+	}
+	defer func() {
+		if err1 := mount.UnmountAll(td, umountflags); err1 != nil && err == nil {
+			err = fmt.Errorf("failed to unmount view: %w", err1)
+		}
+	}()
+
+	if err := fstest.CheckDirectoryEqual(check, td); err != nil {
+		return fmt.Errorf("check directory failed: %w", err)
+	}
+
+	return nil
+}
+
+// checkSnapshots creates a new chain of snapshots in the given snapshotter
+// using the provided appliers, checking each snapshot created in a view
+// against the changes applied to a single directory.
+func checkSnapshots(ctx context.Context, sn snapshots.Snapshotter, work string, as ...fstest.Applier) error {
+	td, err := os.MkdirTemp(work, "flat")
+	if err != nil {
+		return fmt.Errorf("failed to create temp dir: %w", err)
+	}
+	defer os.RemoveAll(td)
+
+	var parentID string
+	for i, a := range as {
+		s, err := createSnapshot(ctx, sn, parentID, work, a)
+		if err != nil {
+			return fmt.Errorf("failed to create snapshot %d: %w", i+1, err)
+		}
+
+		if err := a.Apply(td); err != nil {
+			return fmt.Errorf("failed to apply to check directory on %d: %w", i+1, err)
+		}
+
+		if err := checkSnapshot(ctx, sn, work, s, td); err != nil {
+			return fmt.Errorf("snapshot check failed on snapshot %d: %w", i+1, err)
+		}
+
+		parentID = s
+	}
+	return nil
+
+}
+
+// checkInfo checks that the infos are the same
+func checkInfo(si1, si2 snapshots.Info) error {
+	if si1.Kind != si2.Kind {
+		return fmt.Errorf("expected kind %v, got %v", si1.Kind, si2.Kind)
+	}
+	if si1.Name != si2.Name {
+		return fmt.Errorf("expected name %v, got %v", si1.Name, si2.Name)
+	}
+	if si1.Parent != si2.Parent {
+		return fmt.Errorf("expected Parent %v, got %v", si1.Parent, si2.Parent)
+	}
+	if len(si1.Labels) != len(si2.Labels) {
+		return fmt.Errorf("expected %d labels, got %d", len(si1.Labels), len(si2.Labels))
+	}
+	for k, l1 := range si1.Labels {
+		l2 := si2.Labels[k]
+		if l1 != l2 {
+			return fmt.Errorf("expected label %v, got %v", l1, l2)
+		}
+	}
+	if si1.Created != si2.Created {
+		return fmt.Errorf("expected Created %v, got %v", si1.Created, si2.Created)
+	}
+	if si1.Updated != si2.Updated {
+		return fmt.Errorf("expected Updated %v, got %v", si1.Updated, si2.Updated)
+	}
+
+	return nil
+}

core/snapshots/testsuite/issues.go

@@ -0,0 +1,265 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testsuite
+
+import (
+	"context"
+	"fmt"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/containerd/containerd/v2/core/snapshots"
+	"github.com/containerd/continuity/fs/fstest"
+)
+
+// Checks which cover former issues found in older layering models.
+//
+// NOTE: In older models, applying with tar was used to create read only layers,
+// however with the snapshot model read only layers are created just using
+// mounts and commits. Read write layers are a separate type of snapshot which
+// is not committed, avoiding any confusion in the snapshotter about whether
+// a snapshot will be mutated in the future.
+
+// checkLayerFileUpdate tests the update of a single file in an upper layer
+// Cause of issue was originally related to tar, snapshot should be able to
+// avoid such issues by not relying on tar to create layers.
+// See https://github.com/docker/docker/issues/21555
+func checkLayerFileUpdate(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	l1Init := fstest.Apply(
+		fstest.CreateDir("/etc", 0700),
+		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
+		fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0644),
+	)
+	l2Init := fstest.Apply(
+		fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.2"), 0644),
+		fstest.CreateFile("/etc/profile", []byte("PATH=/usr/bin"), 0666),
+		fstest.CreateDir("/root", 0700),
+		fstest.CreateFile("/root/.bashrc", []byte("PATH=/usr/sbin:/usr/bin"), 0644),
+	)
+
+	var sleepTime time.Duration
+
+	// run 5 times to account for sporadic failure
+	for i := 0; i < 5; i++ {
+		time.Sleep(sleepTime)
+
+		if err := checkSnapshots(ctx, sn, work, l1Init, l2Init); err != nil {
+			t.Fatalf("Check snapshots failed: %+v", err)
+		}
+
+		// Sleep until next second boundary before running again
+		nextTime := time.Now()
+		sleepTime = time.Unix(nextTime.Unix()+1, 0).Sub(nextTime)
+	}
+}
+
+// checkRemoveDirectoryInLowerLayer
+// See https://github.com/docker/docker/issues/25244
+func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	l1Init := fstest.Apply(
+		fstest.CreateDir("/lib", 0700),
+		fstest.CreateFile("/lib/hidden", []byte{}, 0644),
+	)
+	l2Init := fstest.Apply(
+		fstest.RemoveAll("/lib"),
+		fstest.CreateDir("/lib", 0700),
+		fstest.CreateFile("/lib/not-hidden", []byte{}, 0644),
+	)
+	l3Init := fstest.Apply(
+		fstest.CreateFile("/lib/newfile", []byte{}, 0644),
+	)
+
+	if err := checkSnapshots(ctx, sn, work, l1Init, l2Init, l3Init); err != nil {
+		t.Fatalf("Check snapshots failed: %+v", err)
+	}
+}
+
+// checkChown
+// See https://github.com/docker/docker/issues/20240 aufs
+// See https://github.com/docker/docker/issues/24913 overlay
+// see https://github.com/docker/docker/issues/28391 overlay2
+func checkChown(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Chown is not supported on Windows")
+	}
+	l1Init := fstest.Apply(
+		fstest.CreateDir("/opt", 0700),
+		fstest.CreateDir("/opt/a", 0700),
+		fstest.CreateDir("/opt/a/b", 0700),
+		fstest.CreateFile("/opt/a/b/file.txt", []byte("hello"), 0644),
+	)
+	l2Init := fstest.Apply(
+		fstest.Chown("/opt", 1, 1),
+		fstest.Chown("/opt/a", 1, 1),
+		fstest.Chown("/opt/a/b", 1, 1),
+		fstest.Chown("/opt/a/b/file.txt", 1, 1),
+	)
+
+	if err := checkSnapshots(ctx, sn, work, l1Init, l2Init); err != nil {
+		t.Fatalf("Check snapshots failed: %+v", err)
+	}
+}
+
+// checkRename
+// https://github.com/docker/docker/issues/25409
+func checkRename(ss string) func(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	return func(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+		l1Init := fstest.Apply(
+			fstest.CreateDir("/dir1", 0700),
+			fstest.CreateDir("/somefiles", 0700),
+			fstest.CreateFile("/somefiles/f1", []byte("was here first!"), 0644),
+			fstest.CreateFile("/somefiles/f2", []byte("nothing interesting"), 0644),
+		)
+
+		var applier []fstest.Applier
+		switch ss {
+		// With neither OVERLAY_FS_REDIRECT_DIR nor redirect_dir,
+		// renaming the directory on the lower directory doesn't work on overlayfs.
+		// https://github.com/torvalds/linux/blob/v5.18/Documentation/filesystems/overlayfs.rst#renaming-directories
+		//
+		// It doesn't work on fuse-overlayfs either.
+		// https://github.com/containerd/fuse-overlayfs-snapshotter/pull/53#issuecomment-1543442048
+		case "overlayfs", "fuse-overlayfs":
+			// NOP
+		default:
+			applier = append(applier, fstest.Rename("/dir1", "/dir2"))
+		}
+		applier = append(
+			applier,
+			fstest.CreateFile("/somefiles/f1-overwrite", []byte("new content 1"), 0644),
+			fstest.Rename("/somefiles/f1-overwrite", "/somefiles/f1"),
+			fstest.Rename("/somefiles/f2", "/somefiles/f3"),
+		)
+		l2Init := fstest.Apply(applier...)
+
+		if err := checkSnapshots(ctx, sn, work, l1Init, l2Init); err != nil {
+			t.Fatalf("Check snapshots failed: %+v", err)
+		}
+	}
+}
+
+// checkDirectoryPermissionOnCommit
+// https://github.com/docker/docker/issues/27298
+func checkDirectoryPermissionOnCommit(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	if runtime.GOOS == "windows" {
+		t.Skip("Chown is not supported on WCOW")
+	}
+	l1Init := fstest.Apply(
+		fstest.CreateDir("/dir1", 0700),
+		fstest.CreateDir("/dir2", 0700),
+		fstest.CreateDir("/dir3", 0700),
+		fstest.CreateDir("/dir4", 0700),
+		fstest.CreateFile("/dir4/f1", []byte("..."), 0644),
+		fstest.CreateDir("/dir5", 0700),
+		fstest.CreateFile("/dir5/f1", []byte("..."), 0644),
+		fstest.Chown("/dir1", 1, 1),
+		fstest.Chown("/dir2", 1, 1),
+		fstest.Chown("/dir3", 1, 1),
+		fstest.Chown("/dir5", 1, 1),
+		fstest.Chown("/dir5/f1", 1, 1),
+	)
+	l2Init := fstest.Apply(
+		fstest.Chown("/dir2", 0, 0),
+		fstest.RemoveAll("/dir3"),
+		fstest.Chown("/dir4", 1, 1),
+		fstest.Chown("/dir4/f1", 1, 1),
+	)
+	l3Init := fstest.Apply(
+		fstest.CreateDir("/dir3", 0700),
+		fstest.Chown("/dir3", 1, 1),
+		fstest.RemoveAll("/dir5"),
+		fstest.CreateDir("/dir5", 0700),
+		fstest.Chown("/dir5", 1, 1),
+	)
+
+	if err := checkSnapshots(ctx, sn, work, l1Init, l2Init, l3Init); err != nil {
+		t.Fatalf("Check snapshots failed: %+v", err)
+	}
+}
+
+// checkStatInWalk ensures that a stat can be called during a walk
+func checkStatInWalk(ctx context.Context, t *testing.T, sn snapshots.Snapshotter, work string) {
+	prefix := "stats-in-walk-"
+	if err := createNamedSnapshots(ctx, sn, prefix); err != nil {
+		t.Fatal(err)
+	}
+
+	err := sn.Walk(ctx, func(ctx context.Context, si snapshots.Info) error {
+		if !strings.HasPrefix(si.Name, prefix) {
+			// Only stat snapshots from this test
+			return nil
+		}
+		si2, err := sn.Stat(ctx, si.Name)
+		if err != nil {
+			return err
+		}
+
+		return checkInfo(si, si2)
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func createNamedSnapshots(ctx context.Context, snapshotter snapshots.Snapshotter, ns string) error {
+	c1 := fmt.Sprintf("%sc1", ns)
+	c2 := fmt.Sprintf("%sc2", ns)
+	if _, err := snapshotter.Prepare(ctx, c1+"-a", "", opt); err != nil {
+		return err
+	}
+	if err := snapshotter.Commit(ctx, c1, c1+"-a", opt); err != nil {
+		return err
+	}
+	if _, err := snapshotter.Prepare(ctx, c2+"-a", c1, opt); err != nil {
+		return err
+	}
+	if err := snapshotter.Commit(ctx, c2, c2+"-a", opt); err != nil {
+		return err
+	}
+	if _, err := snapshotter.Prepare(ctx, fmt.Sprintf("%sa1", ns), c2, opt); err != nil {
+		return err
+	}
+	if _, err := snapshotter.View(ctx, fmt.Sprintf("%sv1", ns), c2, opt); err != nil {
+		return err
+	}
+	return nil
+}
+
+// More issues to test
+//
+// checkRemoveAfterCommit
+// See https://github.com/docker/docker/issues/24309
+//
+// checkUnixDomainSockets
+// See https://github.com/docker/docker/issues/12080
+//
+// checkDirectoryInodeStability
+// See https://github.com/docker/docker/issues/19647
+//
+// checkOpenFileInodeStability
+// See https://github.com/docker/docker/issues/12327
+//
+// checkGetCWD
+// See https://github.com/docker/docker/issues/19082
+//
+// checkChmod
+// See https://github.com/docker/machine/issues/3327
+//
+// checkRemoveInWalk
+// Allow mutations during walk without deadlocking

core/snapshots/testsuite/testsuite_unix.go

@@ -0,0 +1,30 @@
+//go:build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testsuite
+
+import (
+	"syscall"
+)
+
+func clearMask() func() {
+	oldumask := syscall.Umask(0)
+	return func() {
+		syscall.Umask(oldumask)
+	}
+}

core/snapshots/testsuite/testsuite_windows.go

@@ -0,0 +1,21 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testsuite
+
+func clearMask() func() {
+	return func() {}
+}