Merge pull request #8696 from fuweid/deflaky-blockfile

chore: deflake the blockfile testsuite

go.mod

@@ -11,7 +11,7 @@ require (
 	github.com/containerd/btrfs/v2 v2.0.0
 	github.com/containerd/cgroups/v3 v3.0.1
 	github.com/containerd/console v1.0.3
-	github.com/containerd/continuity v0.4.0
+	github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381
 	github.com/containerd/fifo v1.1.0
 	github.com/containerd/go-cni v1.1.9
 	github.com/containerd/go-runc v1.1.0

go.sum

@@ -232,8 +232,8 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk=
-github.com/containerd/continuity v0.4.0 h1:3LDxKUf4kY/zOUxmdtbxDVYuJQSK+eVg1D/Yk2bbqWQ=
+github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 h1:a5jOuoZHKBi2oH9JsfNqrrPpHhmrYU0NAte3M/EPudw=
-github.com/containerd/continuity v0.4.0/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=

integration/client/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/Microsoft/hcsshim/test v0.0.0-20210408205431-da33ecd607e1
 	github.com/containerd/cgroups/v3 v3.0.1
 	github.com/containerd/containerd v1.7.0 // see replace; the actual version of containerd is replaced with the code at the root of this repository
-	github.com/containerd/continuity v0.4.0
+	github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381
 	github.com/containerd/go-runc v1.1.0
 	github.com/containerd/ttrpc v1.2.2
 	github.com/containerd/typeurl/v2 v2.1.1

integration/client/go.sum

@@ -656,8 +656,8 @@ github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARu
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
 github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM=
-github.com/containerd/continuity v0.4.0 h1:3LDxKUf4kY/zOUxmdtbxDVYuJQSK+eVg1D/Yk2bbqWQ=
+github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 h1:a5jOuoZHKBi2oH9JsfNqrrPpHhmrYU0NAte3M/EPudw=
-github.com/containerd/continuity v0.4.0/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
+github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ=
 github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY=
 github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o=

mount/losetup_linux.go

@@ -89,6 +89,12 @@ func setupLoopDev(backingFile, loopDev string, param LoopParams) (_ *os.File, re
 		return nil, fmt.Errorf("could not set loop fd for device: %s: %w", loopDev, err)
 	}
 
+	defer func() {
+		if retErr != nil {
+			_ = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_CLR_FD, 0)
+		}
+	}()
+
 	// 3. Set Info
 	info := unix.LoopInfo64{}
 	copy(info.File_name[:], backingFile)
@@ -100,27 +106,20 @@ func setupLoopDev(backingFile, loopDev string, param LoopParams) (_ *os.File, re
 		info.Flags |= unix.LO_FLAGS_AUTOCLEAR
 	}
 
-	if param.Direct {
-		info.Flags |= unix.LO_FLAGS_DIRECT_IO
-	}
-
 	err = unix.IoctlLoopSetStatus64(int(loop.Fd()), &info)
-	if err == nil {
+	if err != nil {
-		return loop, nil
+		return nil, fmt.Errorf("failed to set loop device info: %w", err)
 	}
 
+	// 4. Set Direct IO
 	if param.Direct {
-		// Retry w/o direct IO flag in case kernel does not support it. The downside is that
+		err = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_SET_DIRECT_IO, 1)
-		// it will suffer from double cache problem.
+		if err != nil {
-		info.Flags &= ^(uint32(unix.LO_FLAGS_DIRECT_IO))
+			return nil, fmt.Errorf("failed to setup loop with direct: %w", err)
-		err = unix.IoctlLoopSetStatus64(int(loop.Fd()), &info)
-		if err == nil {
-			return loop, nil
 		}
 	}
 
-	_ = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_CLR_FD, 0)
+	return loop, nil
-	return nil, fmt.Errorf("failed to set loop device info: %v", err)
 }
 
 // setupLoop looks for (and possibly creates) a free loop device, and

mount/mount_linux.go

@@ -63,9 +63,6 @@ func (m *Mount) mount(target string) (err error) {
 	}
 
 	flags, data, losetup := parseMountOptions(options)
-	if len(data) > pagesize {
-		return errors.New("mount options is too long")
-	}
 
 	// propagation types.
 	const ptypes = unix.MS_SHARED | unix.MS_PRIVATE | unix.MS_SLAVE | unix.MS_UNBINDABLE
@@ -73,15 +70,27 @@ func (m *Mount) mount(target string) (err error) {
 	// Ensure propagation type change flags aren't included in other calls.
 	oflags := flags &^ ptypes
 
+	var loopParams LoopParams
+	if losetup {
+		loopParams = LoopParams{
+			Readonly:  oflags&unix.MS_RDONLY == unix.MS_RDONLY,
+			Autoclear: true,
+		}
+		loopParams.Direct, data = hasDirectIO(data)
+	}
+
+	dataInStr := strings.Join(data, ",")
+	if len(dataInStr) > pagesize {
+		return errors.New("mount options is too long")
+	}
+
 	// In the case of remounting with changed data (data != ""), need to call mount (moby/moby#34077).
-	if flags&unix.MS_REMOUNT == 0 || data != "" {
+	if flags&unix.MS_REMOUNT == 0 || dataInStr != "" {
 		// Initial call applying all non-propagation flags for mount
 		// or remount with changed data
 		source := m.Source
 		if losetup {
-			loFile, err := setupLoop(m.Source, LoopParams{
+			loFile, err := setupLoop(m.Source, loopParams)
-				Readonly:  oflags&unix.MS_RDONLY == unix.MS_RDONLY,
-				Autoclear: true})
 			if err != nil {
 				return err
 			}
@@ -90,7 +99,7 @@ func (m *Mount) mount(target string) (err error) {
 			// Mount the loop device instead
 			source = loFile.Name()
 		}
-		if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), data); err != nil {
+		if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), dataInStr); err != nil {
 			return err
 		}
 	}
@@ -199,7 +208,7 @@ func UnmountAll(mount string, flags int) error {
 
 // parseMountOptions takes fstab style mount options and parses them for
 // use with a standard mount() syscall
-func parseMountOptions(options []string) (int, string, bool) {
+func parseMountOptions(options []string) (int, []string, bool) {
 	var (
 		flag    int
 		losetup bool
@@ -252,7 +261,16 @@ func parseMountOptions(options []string) (int, string, bool) {
 			data = append(data, o)
 		}
 	}
-	return flag, strings.Join(data, ","), losetup
+	return flag, data, losetup
+}
+
+func hasDirectIO(opts []string) (bool, []string) {
+	for idx, opt := range opts {
+		if opt == "direct-io" {
+			return true, append(opts[:idx], opts[idx+1:]...)
+		}
+	}
+	return false, opts
 }
 
 // compactLowerdirOption updates overlay lowdir option and returns the common

pkg/testutil/helpers.go

@@ -28,6 +28,8 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+const umountflags int = 0
+
 var rootEnabled bool
 
 func init() {

pkg/testutil/mount_linux.go

@@ -1,21 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package testutil
-
-import "golang.org/x/sys/unix"
-
-const umountflags int = unix.MNT_DETACH

pkg/testutil/mount_other.go

@@ -1,22 +0,0 @@
-//go:build !linux
-// +build !linux
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package testutil
-
-const umountflags int = 0

snapshots/blockfile/blockfile.go

@@ -19,6 +19,7 @@ package blockfile
 import (
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
@@ -27,10 +28,11 @@ import (
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/containerd/snapshots/storage"
-
-	"github.com/containerd/continuity/fs"
 )
 
+// viewHookHelper is only used in test for recover the filesystem.
+type viewHookHelper func(backingFile string, fsType string, defaultOpts []string) error
+
 // SnapshotterConfig holds the configurable properties for the blockfile snapshotter
 type SnapshotterConfig struct {
 	// recreateScratch is whether scratch should be recreated even
@@ -44,6 +46,17 @@ type SnapshotterConfig struct {
 
 	// mountOptions are the base options added to the mount (defaults to ["loop"])
 	mountOptions []string
+
+	// testViewHookHelper is used to fsck or mount with rw to handle
+	// the recovery. If we mount ro for view snapshot, we might hit
+	// the issue like
+	//
+	//  (ext4) INFO: recovery required on readonly filesystem
+	//  (ext4) write access unavailable, cannot proceed (try mounting with noload)
+	//
+	// FIXME(fuweid): I don't hit the readonly issue in ssd storage. But it's
+	// easy to reproduce it in slow-storage.
+	testViewHookHelper viewHookHelper
 }
 
 // Opt is an option to configure the overlay snapshotter
@@ -55,7 +68,7 @@ func WithScratchFile(src string) Opt {
 	return func(root string, config *SnapshotterConfig) {
 		config.scratchGenerator = func(dst string) error {
 			// Copy src to dst
-			if err := fs.CopyFile(dst, src); err != nil {
+			if err := copyFileWithSync(dst, src); err != nil {
 				return fmt.Errorf("failed to copy scratch: %w", err)
 			}
 			return nil
@@ -78,12 +91,32 @@ func WithMountOptions(options []string) Opt {
 
 }
 
+// WithRecreateScratch is used to determine that scratch should be recreated
+// even if already exists.
+func WithRecreateScratch(recreate bool) Opt {
+	return func(root string, config *SnapshotterConfig) {
+		config.recreateScratch = recreate
+	}
+}
+
+// withViewHookHelper introduces hook for preparing snapshot for View. It
+// should be used in test only.
+//
+//nolint:nolintlint,unused // not used on all platforms
+func withViewHookHelper(fn viewHookHelper) Opt {
+	return func(_ string, config *SnapshotterConfig) {
+		config.testViewHookHelper = fn
+	}
+}
+
 type snapshotter struct {
 	root    string
 	scratch string
 	fsType  string
 	options []string
 	ms      *storage.MetaStore
+
+	testViewHookHelper viewHookHelper
 }
 
 // NewSnapshotter returns a Snapshotter which copies layers on the underlying
@@ -140,6 +173,8 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) {
 		fsType:  config.fsType,
 		options: config.mountOptions,
 		ms:      ms,
+
+		testViewHookHelper: config.testViewHookHelper,
 	}, nil
 }
 
@@ -343,18 +378,27 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k
 			return fmt.Errorf("failed to create snapshot: %w", err)
 		}
 
+		var path string
 		if len(s.ParentIDs) == 0 || s.Kind == snapshots.KindActive {
-			path := o.getBlockFile(s.ID)
+			path = o.getBlockFile(s.ID)
 
 			if len(s.ParentIDs) > 0 {
-				if err = fs.CopyFile(path, o.getBlockFile(s.ParentIDs[0])); err != nil {
+				if err = copyFileWithSync(path, o.getBlockFile(s.ParentIDs[0])); err != nil {
 					return fmt.Errorf("copying of parent failed: %w", err)
 				}
 			} else {
-				if err = fs.CopyFile(path, o.scratch); err != nil {
+				if err = copyFileWithSync(path, o.scratch); err != nil {
 					return fmt.Errorf("copying of scratch failed: %w", err)
 				}
 			}
+		} else {
+			path = o.getBlockFile(s.ParentIDs[0])
+		}
+
+		if o.testViewHookHelper != nil {
+			if err := o.testViewHookHelper(path, o.fsType, o.options); err != nil {
+				return fmt.Errorf("failed to handle the viewHookHelper: %w", err)
+			}
 		}
 
 		return nil
@@ -401,3 +445,20 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount {
 func (o *snapshotter) Close() error {
 	return o.ms.Close()
 }
+
+func copyFileWithSync(target, source string) error {
+	src, err := os.Open(source)
+	if err != nil {
+		return fmt.Errorf("failed to open source %s: %w", source, err)
+	}
+	defer src.Close()
+	tgt, err := os.Create(target)
+	if err != nil {
+		return fmt.Errorf("failed to open target %s: %w", target, err)
+	}
+	defer tgt.Close()
+	defer tgt.Sync()
+
+	_, err = io.Copy(tgt, src)
+	return err
+}

snapshots/blockfile/blockfile_loopsetup_test.go

@@ -26,9 +26,6 @@ import (
 	"testing"
 
 	"github.com/containerd/containerd/mount"
-	"github.com/containerd/continuity/fs"
-	"github.com/containerd/continuity/testutil/loopback"
-	"golang.org/x/sys/unix"
 )
 
 func setupSnapshotter(t *testing.T) ([]Opt, error) {
@@ -37,52 +34,94 @@ func setupSnapshotter(t *testing.T) ([]Opt, error) {
 		t.Skipf("Could not find mkfs.ext4: %v", err)
 	}
 
-	loopbackSize := int64(128 << 20) // 128 MB
+	loopbackSize := int64(8 << 20) // 8 MB
 	if os.Getpagesize() > 4096 {
 		loopbackSize = int64(650 << 20) // 650 MB
 	}
 
-	loop, err := loopback.New(loopbackSize)
+	scratch := filepath.Join(t.TempDir(), "scratch")
+	scratchDevFile, err := os.OpenFile(scratch, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create %s: %w", scratch, err)
 	}
-	defer loop.Close()
 
-	if out, err := exec.Command(mkfs, loop.Device).CombinedOutput(); err != nil {
+	if err := scratchDevFile.Truncate(loopbackSize); err != nil {
+		scratchDevFile.Close()
+		return nil, fmt.Errorf("failed to resize %s file: %w", scratch, err)
+	}
+
+	if err := scratchDevFile.Sync(); err != nil {
+		scratchDevFile.Close()
+		return nil, fmt.Errorf("failed to sync %s file: %w", scratch, err)
+	}
+	scratchDevFile.Close()
+
+	if out, err := exec.Command(mkfs, scratch).CombinedOutput(); err != nil {
 		return nil, fmt.Errorf("failed to make ext4 filesystem (out: %q): %w", out, err)
 	}
-	// sync after a mkfs on the loopback before trying to mount the device
-	unix.Sync()
 
-	if err := testMount(t, loop.Device); err != nil {
+	defaultOpts := []string{"loop", "direct-io", "sync"}
-		return nil, err
-	}
 
-	scratch := filepath.Join(t.TempDir(), "scratch")
+	if err := testMount(t, scratch, defaultOpts); err != nil {
-	err = fs.CopyFile(scratch, loop.File)
-	if err != nil {
 		return nil, err
 	}
 
 	return []Opt{
 		WithScratchFile(scratch),
 		WithFSType("ext4"),
-		WithMountOptions([]string{"loop", "sync"}),
+		WithMountOptions(defaultOpts),
+		WithRecreateScratch(false), // reduce IO presure in CI
+		withViewHookHelper(testViewHook),
 	}, nil
 }
 
-func testMount(t *testing.T, device string) error {
+func testMount(t *testing.T, scratchFile string, opts []string) error {
 	root, err := os.MkdirTemp(t.TempDir(), "")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(root)
 
-	if out, err := exec.Command("mount", device, root).CombinedOutput(); err != nil {
+	m := []mount.Mount{
-		return fmt.Errorf("failed to mount device %s (out: %q): %w", device, out, err)
+		{
+			Type:    "ext4",
+			Source:  scratchFile,
+			Options: opts,
+		},
 	}
+
+	if err := mount.All(m, root); err != nil {
+		return fmt.Errorf("failed to mount device %s: %w", scratchFile, err)
+	}
+
 	if err := os.Remove(filepath.Join(root, "lost+found")); err != nil {
 		return err
 	}
-	return mount.UnmountAll(root, unix.MNT_DETACH)
+	return mount.UnmountAll(root, 0)
+}
+
+func testViewHook(backingFile string, fsType string, defaultOpts []string) error {
+	root, err := os.MkdirTemp("", "blockfile-testViewHook-XXXX")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(root)
+
+	// FIXME(fuweid): Mount with rw to force fs to handle recover
+	mountOpts := []mount.Mount{
+		{
+			Type:    fsType,
+			Source:  backingFile,
+			Options: defaultOpts,
+		},
+	}
+
+	if err := mount.All(mountOpts, root); err != nil {
+		return fmt.Errorf("failed to mount device %s: %w", backingFile, err)
+	}
+
+	if err := mount.UnmountAll(root, 0); err != nil {
+		return fmt.Errorf("failed to unmount device %s: %w", backingFile, err)
+	}
+	return nil
 }

snapshots/btrfs/btrfs_test.go

@@ -102,7 +102,7 @@ func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshots.Snap
 			if err := snapshotter.Close(); err != nil {
 				return err
 			}
-			err := mount.UnmountAll(root, unix.MNT_DETACH)
+			err := mount.UnmountAll(root, 0)
 			if cerr := loop.Close(); cerr != nil {
 				err = fmt.Errorf("device cleanup failed: %w", cerr)
 			}

snapshots/testsuite/helpers.go

@@ -27,6 +27,8 @@ import (
 	"github.com/containerd/continuity/fs/fstest"
 )
 
+const umountflags int = 0
+
 func applyToMounts(m []mount.Mount, work string, a fstest.Applier) (err error) {
 	td, err := os.MkdirTemp(work, "prepare")
 	if err != nil {

snapshots/testsuite/helpers_linux.go

@@ -1,21 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package testsuite
-
-import "golang.org/x/sys/unix"
-
-const umountflags int = unix.MNT_DETACH

snapshots/testsuite/helpers_other.go

@@ -1,21 +0,0 @@
-//go:build !linux
-
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package testsuite
-
-const umountflags int = 0

snapshots/testsuite/testsuite.go

@@ -959,7 +959,6 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn
 				t.Fatalf("[layer %d] preparing doesn't equal to flat after apply: %+v", i, err)
 			}
 
-			sync()
 			testutil.Unmount(t, preparing)
 
 			parent = filepath.Join(work, fmt.Sprintf("committed-%d", i))

snapshots/testsuite/testsuite_unix.go

@@ -20,8 +20,6 @@ package testsuite
 
 import (
 	"syscall"
-
-	"golang.org/x/sys/unix"
 )
 
 func clearMask() func() {
@@ -30,7 +28,3 @@ func clearMask() func() {
 		syscall.Umask(oldumask)
 	}
 }
-
-func sync() {
-	unix.Sync()
-}

snapshots/testsuite/testsuite_windows.go

@@ -19,5 +19,3 @@ package testsuite
 func clearMask() func() {
 	return func() {}
 }
-
-func sync() {}

vendor/github.com/containerd/continuity/fs/copy.go

@@ -18,20 +18,13 @@ package fs
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
-	"sync"
 
 	"github.com/sirupsen/logrus"
 )
 
-var bufferPool = &sync.Pool{
-	New: func() interface{} {
-		buffer := make([]byte, 32*1024)
-		return &buffer
-	},
-}
-
 // XAttrErrorHandler transform a non-nil xattr error.
 // Return nil to ignore an error.
 // xattrKey can be empty for listxattr operation.
@@ -199,5 +192,6 @@ func openAndCopyFile(target, source string) error {
 	}
 	defer tgt.Close()
 
-	return copyFileContent(tgt, src)
+	_, err = io.Copy(tgt, src)
+	return err
 }

vendor/github.com/containerd/continuity/fs/copy_darwin.go

@@ -25,7 +25,7 @@ import (
 
 func copyFile(target, source string) error {
 	if err := unix.Clonefile(source, target, unix.CLONE_NOFOLLOW); err != nil {
-		if !errors.Is(err, unix.ENOTSUP) {
+		if !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EXDEV) {
 			return fmt.Errorf("clonefile failed: %w", err)
 		}
 

vendor/github.com/containerd/continuity/fs/copy_linux.go

@@ -18,7 +18,6 @@ package fs
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"syscall"
 
@@ -62,51 +61,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error {
 	return nil
 }
 
-const maxSSizeT = int64(^uint(0) >> 1)
-
-func copyFileContent(dst, src *os.File) error {
-	st, err := src.Stat()
-	if err != nil {
-		return fmt.Errorf("unable to stat source: %w", err)
-	}
-
-	size := st.Size()
-	first := true
-	srcFd := int(src.Fd())
-	dstFd := int(dst.Fd())
-
-	for size > 0 {
-		// Ensure that we are never trying to copy more than SSIZE_MAX at a
-		// time and at the same time avoids overflows when the file is larger
-		// than 4GB on 32-bit systems.
-		var copySize int
-		if size > maxSSizeT {
-			copySize = int(maxSSizeT)
-		} else {
-			copySize = int(size)
-		}
-		n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, copySize, 0)
-		if err != nil {
-			if (err != unix.ENOSYS && err != unix.EXDEV) || !first {
-				return fmt.Errorf("copy file range failed: %w", err)
-			}
-
-			buf := bufferPool.Get().(*[]byte)
-			_, err = io.CopyBuffer(dst, src, *buf)
-			bufferPool.Put(buf)
-			if err != nil {
-				return fmt.Errorf("userspace copy failed: %w", err)
-			}
-			return nil
-		}
-
-		first = false
-		size -= int64(n)
-	}
-
-	return nil
-}
-
 func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
 	xattrKeys, err := sysx.LListxattr(src)
 	if err != nil {

vendor/github.com/containerd/continuity/fs/copy_unix.go

@@ -21,7 +21,6 @@ package fs
 
 import (
 	"fmt"
-	"io"
 	"os"
 	"runtime"
 	"syscall"
@@ -61,14 +60,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error {
 	return nil
 }
 
-func copyFileContent(dst, src *os.File) error {
-	buf := bufferPool.Get().(*[]byte)
-	_, err := io.CopyBuffer(dst, src, *buf)
-	bufferPool.Put(buf)
-
-	return err
-}
-
 func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
 	xattrKeys, err := sysx.LListxattr(src)
 	if err != nil {

vendor/github.com/containerd/continuity/fs/copy_windows.go

@@ -19,7 +19,6 @@ package fs
 import (
 	"errors"
 	"fmt"
-	"io"
 	"os"
 
 	winio "github.com/Microsoft/go-winio"
@@ -72,13 +71,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error {
 	return nil
 }
 
-func copyFileContent(dst, src *os.File) error {
-	buf := bufferPool.Get().(*[]byte)
-	_, err := io.CopyBuffer(dst, src, *buf)
-	bufferPool.Put(buf)
-	return err
-}
-
 func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error {
 	return nil
 }

vendor/github.com/containerd/continuity/fs/fstest/file.go

@@ -65,7 +65,12 @@ func writeFileStream(name string, stream func() io.Reader, perm os.FileMode) App
 			return err
 		}
 		defer func() {
-			err := f.Close()
+			err := f.Sync()
+			if err != nil && retErr == nil {
+				retErr = err
+			}
+
+			err = f.Close()
 			if err != nil && retErr == nil {
 				retErr = err
 			}

vendor/modules.txt

@@ -98,7 +98,7 @@ github.com/containerd/cgroups/v3/cgroup2/stats
 # github.com/containerd/console v1.0.3
 ## explicit; go 1.13
 github.com/containerd/console
-# github.com/containerd/continuity v0.4.0
+# github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381
 ## explicit; go 1.19
 github.com/containerd/continuity
 github.com/containerd/continuity/devices