diff --git a/go.mod b/go.mod index e894c94c4..d7cb3c6b4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/containerd/btrfs/v2 v2.0.0 github.com/containerd/cgroups/v3 v3.0.1 github.com/containerd/console v1.0.3 - github.com/containerd/continuity v0.4.0 + github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 github.com/containerd/fifo v1.1.0 github.com/containerd/go-cni v1.1.9 github.com/containerd/go-runc v1.1.0 diff --git a/go.sum b/go.sum index 6cbb0331f..66008c9c6 100644 --- a/go.sum +++ b/go.sum @@ -232,8 +232,8 @@ github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.2.2/go.mod h1:pWygW9u7LtS1o4N/Tn0FoCFDIXZ7rxcMX7HX1Dmibvk= -github.com/containerd/continuity v0.4.0 h1:3LDxKUf4kY/zOUxmdtbxDVYuJQSK+eVg1D/Yk2bbqWQ= -github.com/containerd/continuity v0.4.0/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 h1:a5jOuoZHKBi2oH9JsfNqrrPpHhmrYU0NAte3M/EPudw= +github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= diff --git a/integration/client/go.mod b/integration/client/go.mod index 4404d56bc..8bc864ea4 100644 --- a/integration/client/go.mod +++ b/integration/client/go.mod @@ -8,7 +8,7 @@ require ( github.com/Microsoft/hcsshim/test v0.0.0-20210408205431-da33ecd607e1 github.com/containerd/cgroups/v3 v3.0.1 github.com/containerd/containerd v1.7.0 // see replace; the actual version of containerd is replaced with the code at the root of this repository - github.com/containerd/continuity v0.4.0 + github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 github.com/containerd/go-runc v1.1.0 github.com/containerd/ttrpc v1.2.2 github.com/containerd/typeurl/v2 v2.1.1 diff --git a/integration/client/go.sum b/integration/client/go.sum index 4c4bdbf6d..3c127d40a 100644 --- a/integration/client/go.sum +++ b/integration/client/go.sum @@ -656,8 +656,8 @@ github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARu github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= -github.com/containerd/continuity v0.4.0 h1:3LDxKUf4kY/zOUxmdtbxDVYuJQSK+eVg1D/Yk2bbqWQ= -github.com/containerd/continuity v0.4.0/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 h1:a5jOuoZHKBi2oH9JsfNqrrPpHhmrYU0NAte3M/EPudw= +github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= diff --git a/mount/losetup_linux.go b/mount/losetup_linux.go index 567ebb4e9..e3647e954 100644 --- a/mount/losetup_linux.go +++ b/mount/losetup_linux.go @@ -89,6 +89,12 @@ func setupLoopDev(backingFile, loopDev string, param LoopParams) (_ *os.File, re return nil, fmt.Errorf("could not set loop fd for device: %s: %w", loopDev, err) } + defer func() { + if retErr != nil { + _ = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_CLR_FD, 0) + } + }() + // 3. Set Info info := unix.LoopInfo64{} copy(info.File_name[:], backingFile) @@ -100,27 +106,20 @@ func setupLoopDev(backingFile, loopDev string, param LoopParams) (_ *os.File, re info.Flags |= unix.LO_FLAGS_AUTOCLEAR } - if param.Direct { - info.Flags |= unix.LO_FLAGS_DIRECT_IO - } - err = unix.IoctlLoopSetStatus64(int(loop.Fd()), &info) - if err == nil { - return loop, nil + if err != nil { + return nil, fmt.Errorf("failed to set loop device info: %w", err) } + // 4. Set Direct IO if param.Direct { - // Retry w/o direct IO flag in case kernel does not support it. The downside is that - // it will suffer from double cache problem. - info.Flags &= ^(uint32(unix.LO_FLAGS_DIRECT_IO)) - err = unix.IoctlLoopSetStatus64(int(loop.Fd()), &info) - if err == nil { - return loop, nil + err = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_SET_DIRECT_IO, 1) + if err != nil { + return nil, fmt.Errorf("failed to setup loop with direct: %w", err) } } - _ = unix.IoctlSetInt(int(loop.Fd()), unix.LOOP_CLR_FD, 0) - return nil, fmt.Errorf("failed to set loop device info: %v", err) + return loop, nil } // setupLoop looks for (and possibly creates) a free loop device, and diff --git a/mount/mount_linux.go b/mount/mount_linux.go index 736bb4cbb..adb59b819 100644 --- a/mount/mount_linux.go +++ b/mount/mount_linux.go @@ -63,9 +63,6 @@ func (m *Mount) mount(target string) (err error) { } flags, data, losetup := parseMountOptions(options) - if len(data) > pagesize { - return errors.New("mount options is too long") - } // propagation types. const ptypes = unix.MS_SHARED | unix.MS_PRIVATE | unix.MS_SLAVE | unix.MS_UNBINDABLE @@ -73,15 +70,27 @@ func (m *Mount) mount(target string) (err error) { // Ensure propagation type change flags aren't included in other calls. oflags := flags &^ ptypes + var loopParams LoopParams + if losetup { + loopParams = LoopParams{ + Readonly: oflags&unix.MS_RDONLY == unix.MS_RDONLY, + Autoclear: true, + } + loopParams.Direct, data = hasDirectIO(data) + } + + dataInStr := strings.Join(data, ",") + if len(dataInStr) > pagesize { + return errors.New("mount options is too long") + } + // In the case of remounting with changed data (data != ""), need to call mount (moby/moby#34077). - if flags&unix.MS_REMOUNT == 0 || data != "" { + if flags&unix.MS_REMOUNT == 0 || dataInStr != "" { // Initial call applying all non-propagation flags for mount // or remount with changed data source := m.Source if losetup { - loFile, err := setupLoop(m.Source, LoopParams{ - Readonly: oflags&unix.MS_RDONLY == unix.MS_RDONLY, - Autoclear: true}) + loFile, err := setupLoop(m.Source, loopParams) if err != nil { return err } @@ -90,7 +99,7 @@ func (m *Mount) mount(target string) (err error) { // Mount the loop device instead source = loFile.Name() } - if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), data); err != nil { + if err := mountAt(chdir, source, target, m.Type, uintptr(oflags), dataInStr); err != nil { return err } } @@ -199,7 +208,7 @@ func UnmountAll(mount string, flags int) error { // parseMountOptions takes fstab style mount options and parses them for // use with a standard mount() syscall -func parseMountOptions(options []string) (int, string, bool) { +func parseMountOptions(options []string) (int, []string, bool) { var ( flag int losetup bool @@ -252,7 +261,16 @@ func parseMountOptions(options []string) (int, string, bool) { data = append(data, o) } } - return flag, strings.Join(data, ","), losetup + return flag, data, losetup +} + +func hasDirectIO(opts []string) (bool, []string) { + for idx, opt := range opts { + if opt == "direct-io" { + return true, append(opts[:idx], opts[idx+1:]...) + } + } + return false, opts } // compactLowerdirOption updates overlay lowdir option and returns the common diff --git a/pkg/testutil/helpers.go b/pkg/testutil/helpers.go index 850f51f83..81bf54b24 100644 --- a/pkg/testutil/helpers.go +++ b/pkg/testutil/helpers.go @@ -28,6 +28,8 @@ import ( "github.com/stretchr/testify/assert" ) +const umountflags int = 0 + var rootEnabled bool func init() { diff --git a/pkg/testutil/mount_linux.go b/pkg/testutil/mount_linux.go deleted file mode 100644 index 9c4cf8c16..000000000 --- a/pkg/testutil/mount_linux.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package testutil - -import "golang.org/x/sys/unix" - -const umountflags int = unix.MNT_DETACH diff --git a/pkg/testutil/mount_other.go b/pkg/testutil/mount_other.go deleted file mode 100644 index b1aa7d460..000000000 --- a/pkg/testutil/mount_other.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build !linux -// +build !linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package testutil - -const umountflags int = 0 diff --git a/snapshots/blockfile/blockfile.go b/snapshots/blockfile/blockfile.go index dd6257936..f1e376ae3 100644 --- a/snapshots/blockfile/blockfile.go +++ b/snapshots/blockfile/blockfile.go @@ -19,6 +19,7 @@ package blockfile import ( "context" "fmt" + "io" "os" "path/filepath" @@ -27,10 +28,11 @@ import ( "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/snapshots" "github.com/containerd/containerd/snapshots/storage" - - "github.com/containerd/continuity/fs" ) +// viewHookHelper is only used in test for recover the filesystem. +type viewHookHelper func(backingFile string, fsType string, defaultOpts []string) error + // SnapshotterConfig holds the configurable properties for the blockfile snapshotter type SnapshotterConfig struct { // recreateScratch is whether scratch should be recreated even @@ -44,6 +46,17 @@ type SnapshotterConfig struct { // mountOptions are the base options added to the mount (defaults to ["loop"]) mountOptions []string + + // testViewHookHelper is used to fsck or mount with rw to handle + // the recovery. If we mount ro for view snapshot, we might hit + // the issue like + // + // (ext4) INFO: recovery required on readonly filesystem + // (ext4) write access unavailable, cannot proceed (try mounting with noload) + // + // FIXME(fuweid): I don't hit the readonly issue in ssd storage. But it's + // easy to reproduce it in slow-storage. + testViewHookHelper viewHookHelper } // Opt is an option to configure the overlay snapshotter @@ -55,7 +68,7 @@ func WithScratchFile(src string) Opt { return func(root string, config *SnapshotterConfig) { config.scratchGenerator = func(dst string) error { // Copy src to dst - if err := fs.CopyFile(dst, src); err != nil { + if err := copyFileWithSync(dst, src); err != nil { return fmt.Errorf("failed to copy scratch: %w", err) } return nil @@ -78,12 +91,32 @@ func WithMountOptions(options []string) Opt { } +// WithRecreateScratch is used to determine that scratch should be recreated +// even if already exists. +func WithRecreateScratch(recreate bool) Opt { + return func(root string, config *SnapshotterConfig) { + config.recreateScratch = recreate + } +} + +// withViewHookHelper introduces hook for preparing snapshot for View. It +// should be used in test only. +// +//nolint:nolintlint,unused // not used on all platforms +func withViewHookHelper(fn viewHookHelper) Opt { + return func(_ string, config *SnapshotterConfig) { + config.testViewHookHelper = fn + } +} + type snapshotter struct { root string scratch string fsType string options []string ms *storage.MetaStore + + testViewHookHelper viewHookHelper } // NewSnapshotter returns a Snapshotter which copies layers on the underlying @@ -140,6 +173,8 @@ func NewSnapshotter(root string, opts ...Opt) (snapshots.Snapshotter, error) { fsType: config.fsType, options: config.mountOptions, ms: ms, + + testViewHookHelper: config.testViewHookHelper, }, nil } @@ -343,18 +378,27 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, k return fmt.Errorf("failed to create snapshot: %w", err) } + var path string if len(s.ParentIDs) == 0 || s.Kind == snapshots.KindActive { - path := o.getBlockFile(s.ID) + path = o.getBlockFile(s.ID) if len(s.ParentIDs) > 0 { - if err = fs.CopyFile(path, o.getBlockFile(s.ParentIDs[0])); err != nil { + if err = copyFileWithSync(path, o.getBlockFile(s.ParentIDs[0])); err != nil { return fmt.Errorf("copying of parent failed: %w", err) } } else { - if err = fs.CopyFile(path, o.scratch); err != nil { + if err = copyFileWithSync(path, o.scratch); err != nil { return fmt.Errorf("copying of scratch failed: %w", err) } } + } else { + path = o.getBlockFile(s.ParentIDs[0]) + } + + if o.testViewHookHelper != nil { + if err := o.testViewHookHelper(path, o.fsType, o.options); err != nil { + return fmt.Errorf("failed to handle the viewHookHelper: %w", err) + } } return nil @@ -401,3 +445,20 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount { func (o *snapshotter) Close() error { return o.ms.Close() } + +func copyFileWithSync(target, source string) error { + src, err := os.Open(source) + if err != nil { + return fmt.Errorf("failed to open source %s: %w", source, err) + } + defer src.Close() + tgt, err := os.Create(target) + if err != nil { + return fmt.Errorf("failed to open target %s: %w", target, err) + } + defer tgt.Close() + defer tgt.Sync() + + _, err = io.Copy(tgt, src) + return err +} diff --git a/snapshots/blockfile/blockfile_loopsetup_test.go b/snapshots/blockfile/blockfile_loopsetup_test.go index 06b4dd306..bda55f58b 100644 --- a/snapshots/blockfile/blockfile_loopsetup_test.go +++ b/snapshots/blockfile/blockfile_loopsetup_test.go @@ -26,9 +26,6 @@ import ( "testing" "github.com/containerd/containerd/mount" - "github.com/containerd/continuity/fs" - "github.com/containerd/continuity/testutil/loopback" - "golang.org/x/sys/unix" ) func setupSnapshotter(t *testing.T) ([]Opt, error) { @@ -37,52 +34,94 @@ func setupSnapshotter(t *testing.T) ([]Opt, error) { t.Skipf("Could not find mkfs.ext4: %v", err) } - loopbackSize := int64(128 << 20) // 128 MB + loopbackSize := int64(8 << 20) // 8 MB if os.Getpagesize() > 4096 { loopbackSize = int64(650 << 20) // 650 MB } - loop, err := loopback.New(loopbackSize) + scratch := filepath.Join(t.TempDir(), "scratch") + scratchDevFile, err := os.OpenFile(scratch, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create %s: %w", scratch, err) } - defer loop.Close() - if out, err := exec.Command(mkfs, loop.Device).CombinedOutput(); err != nil { + if err := scratchDevFile.Truncate(loopbackSize); err != nil { + scratchDevFile.Close() + return nil, fmt.Errorf("failed to resize %s file: %w", scratch, err) + } + + if err := scratchDevFile.Sync(); err != nil { + scratchDevFile.Close() + return nil, fmt.Errorf("failed to sync %s file: %w", scratch, err) + } + scratchDevFile.Close() + + if out, err := exec.Command(mkfs, scratch).CombinedOutput(); err != nil { return nil, fmt.Errorf("failed to make ext4 filesystem (out: %q): %w", out, err) } - // sync after a mkfs on the loopback before trying to mount the device - unix.Sync() - if err := testMount(t, loop.Device); err != nil { - return nil, err - } + defaultOpts := []string{"loop", "direct-io", "sync"} - scratch := filepath.Join(t.TempDir(), "scratch") - err = fs.CopyFile(scratch, loop.File) - if err != nil { + if err := testMount(t, scratch, defaultOpts); err != nil { return nil, err } return []Opt{ WithScratchFile(scratch), WithFSType("ext4"), - WithMountOptions([]string{"loop", "sync"}), + WithMountOptions(defaultOpts), + WithRecreateScratch(false), // reduce IO presure in CI + withViewHookHelper(testViewHook), }, nil } -func testMount(t *testing.T, device string) error { +func testMount(t *testing.T, scratchFile string, opts []string) error { root, err := os.MkdirTemp(t.TempDir(), "") if err != nil { return err } defer os.RemoveAll(root) - if out, err := exec.Command("mount", device, root).CombinedOutput(); err != nil { - return fmt.Errorf("failed to mount device %s (out: %q): %w", device, out, err) + m := []mount.Mount{ + { + Type: "ext4", + Source: scratchFile, + Options: opts, + }, } + + if err := mount.All(m, root); err != nil { + return fmt.Errorf("failed to mount device %s: %w", scratchFile, err) + } + if err := os.Remove(filepath.Join(root, "lost+found")); err != nil { return err } - return mount.UnmountAll(root, unix.MNT_DETACH) + return mount.UnmountAll(root, 0) +} + +func testViewHook(backingFile string, fsType string, defaultOpts []string) error { + root, err := os.MkdirTemp("", "blockfile-testViewHook-XXXX") + if err != nil { + return err + } + defer os.RemoveAll(root) + + // FIXME(fuweid): Mount with rw to force fs to handle recover + mountOpts := []mount.Mount{ + { + Type: fsType, + Source: backingFile, + Options: defaultOpts, + }, + } + + if err := mount.All(mountOpts, root); err != nil { + return fmt.Errorf("failed to mount device %s: %w", backingFile, err) + } + + if err := mount.UnmountAll(root, 0); err != nil { + return fmt.Errorf("failed to unmount device %s: %w", backingFile, err) + } + return nil } diff --git a/snapshots/btrfs/btrfs_test.go b/snapshots/btrfs/btrfs_test.go index 0671d15c4..02eb93d37 100644 --- a/snapshots/btrfs/btrfs_test.go +++ b/snapshots/btrfs/btrfs_test.go @@ -102,7 +102,7 @@ func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshots.Snap if err := snapshotter.Close(); err != nil { return err } - err := mount.UnmountAll(root, unix.MNT_DETACH) + err := mount.UnmountAll(root, 0) if cerr := loop.Close(); cerr != nil { err = fmt.Errorf("device cleanup failed: %w", cerr) } diff --git a/snapshots/testsuite/helpers.go b/snapshots/testsuite/helpers.go index 4315db8ff..8032418fc 100644 --- a/snapshots/testsuite/helpers.go +++ b/snapshots/testsuite/helpers.go @@ -27,6 +27,8 @@ import ( "github.com/containerd/continuity/fs/fstest" ) +const umountflags int = 0 + func applyToMounts(m []mount.Mount, work string, a fstest.Applier) (err error) { td, err := os.MkdirTemp(work, "prepare") if err != nil { diff --git a/snapshots/testsuite/helpers_linux.go b/snapshots/testsuite/helpers_linux.go deleted file mode 100644 index 9e4aac356..000000000 --- a/snapshots/testsuite/helpers_linux.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package testsuite - -import "golang.org/x/sys/unix" - -const umountflags int = unix.MNT_DETACH diff --git a/snapshots/testsuite/helpers_other.go b/snapshots/testsuite/helpers_other.go deleted file mode 100644 index d39ec0734..000000000 --- a/snapshots/testsuite/helpers_other.go +++ /dev/null @@ -1,21 +0,0 @@ -//go:build !linux - -/* - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package testsuite - -const umountflags int = 0 diff --git a/snapshots/testsuite/testsuite.go b/snapshots/testsuite/testsuite.go index 7afbd953e..36982adab 100644 --- a/snapshots/testsuite/testsuite.go +++ b/snapshots/testsuite/testsuite.go @@ -959,7 +959,6 @@ func check128LayersMount(name string) func(ctx context.Context, t *testing.T, sn t.Fatalf("[layer %d] preparing doesn't equal to flat after apply: %+v", i, err) } - sync() testutil.Unmount(t, preparing) parent = filepath.Join(work, fmt.Sprintf("committed-%d", i)) diff --git a/snapshots/testsuite/testsuite_unix.go b/snapshots/testsuite/testsuite_unix.go index 8a274ce24..e37cf86a1 100644 --- a/snapshots/testsuite/testsuite_unix.go +++ b/snapshots/testsuite/testsuite_unix.go @@ -20,8 +20,6 @@ package testsuite import ( "syscall" - - "golang.org/x/sys/unix" ) func clearMask() func() { @@ -30,7 +28,3 @@ func clearMask() func() { syscall.Umask(oldumask) } } - -func sync() { - unix.Sync() -} diff --git a/snapshots/testsuite/testsuite_windows.go b/snapshots/testsuite/testsuite_windows.go index 382a85978..a3cb82fbf 100644 --- a/snapshots/testsuite/testsuite_windows.go +++ b/snapshots/testsuite/testsuite_windows.go @@ -19,5 +19,3 @@ package testsuite func clearMask() func() { return func() {} } - -func sync() {} diff --git a/vendor/github.com/containerd/continuity/fs/copy.go b/vendor/github.com/containerd/continuity/fs/copy.go index a1cae0f95..af3abdd4c 100644 --- a/vendor/github.com/containerd/continuity/fs/copy.go +++ b/vendor/github.com/containerd/continuity/fs/copy.go @@ -18,20 +18,13 @@ package fs import ( "fmt" + "io" "os" "path/filepath" - "sync" "github.com/sirupsen/logrus" ) -var bufferPool = &sync.Pool{ - New: func() interface{} { - buffer := make([]byte, 32*1024) - return &buffer - }, -} - // XAttrErrorHandler transform a non-nil xattr error. // Return nil to ignore an error. // xattrKey can be empty for listxattr operation. @@ -199,5 +192,6 @@ func openAndCopyFile(target, source string) error { } defer tgt.Close() - return copyFileContent(tgt, src) + _, err = io.Copy(tgt, src) + return err } diff --git a/vendor/github.com/containerd/continuity/fs/copy_darwin.go b/vendor/github.com/containerd/continuity/fs/copy_darwin.go index 73e5af1ec..97fc2e8ea 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_darwin.go +++ b/vendor/github.com/containerd/continuity/fs/copy_darwin.go @@ -25,7 +25,7 @@ import ( func copyFile(target, source string) error { if err := unix.Clonefile(source, target, unix.CLONE_NOFOLLOW); err != nil { - if !errors.Is(err, unix.ENOTSUP) { + if !errors.Is(err, unix.ENOTSUP) && !errors.Is(err, unix.EXDEV) { return fmt.Errorf("clonefile failed: %w", err) } diff --git a/vendor/github.com/containerd/continuity/fs/copy_linux.go b/vendor/github.com/containerd/continuity/fs/copy_linux.go index 1906e5e01..48ac3fbd3 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_linux.go +++ b/vendor/github.com/containerd/continuity/fs/copy_linux.go @@ -18,7 +18,6 @@ package fs import ( "fmt" - "io" "os" "syscall" @@ -62,51 +61,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error { return nil } -const maxSSizeT = int64(^uint(0) >> 1) - -func copyFileContent(dst, src *os.File) error { - st, err := src.Stat() - if err != nil { - return fmt.Errorf("unable to stat source: %w", err) - } - - size := st.Size() - first := true - srcFd := int(src.Fd()) - dstFd := int(dst.Fd()) - - for size > 0 { - // Ensure that we are never trying to copy more than SSIZE_MAX at a - // time and at the same time avoids overflows when the file is larger - // than 4GB on 32-bit systems. - var copySize int - if size > maxSSizeT { - copySize = int(maxSSizeT) - } else { - copySize = int(size) - } - n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, copySize, 0) - if err != nil { - if (err != unix.ENOSYS && err != unix.EXDEV) || !first { - return fmt.Errorf("copy file range failed: %w", err) - } - - buf := bufferPool.Get().(*[]byte) - _, err = io.CopyBuffer(dst, src, *buf) - bufferPool.Put(buf) - if err != nil { - return fmt.Errorf("userspace copy failed: %w", err) - } - return nil - } - - first = false - size -= int64(n) - } - - return nil -} - func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error { xattrKeys, err := sysx.LListxattr(src) if err != nil { diff --git a/vendor/github.com/containerd/continuity/fs/copy_unix.go b/vendor/github.com/containerd/continuity/fs/copy_unix.go index dd957872b..1311250e1 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_unix.go +++ b/vendor/github.com/containerd/continuity/fs/copy_unix.go @@ -21,7 +21,6 @@ package fs import ( "fmt" - "io" "os" "runtime" "syscall" @@ -61,14 +60,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error { return nil } -func copyFileContent(dst, src *os.File) error { - buf := bufferPool.Get().(*[]byte) - _, err := io.CopyBuffer(dst, src, *buf) - bufferPool.Put(buf) - - return err -} - func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error { xattrKeys, err := sysx.LListxattr(src) if err != nil { diff --git a/vendor/github.com/containerd/continuity/fs/copy_windows.go b/vendor/github.com/containerd/continuity/fs/copy_windows.go index 6ec13b989..1fad4c3ad 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_windows.go +++ b/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -19,7 +19,6 @@ package fs import ( "errors" "fmt" - "io" "os" winio "github.com/Microsoft/go-winio" @@ -72,13 +71,6 @@ func copyFileInfo(fi os.FileInfo, src, name string) error { return nil } -func copyFileContent(dst, src *os.File) error { - buf := bufferPool.Get().(*[]byte) - _, err := io.CopyBuffer(dst, src, *buf) - bufferPool.Put(buf) - return err -} - func copyXAttrs(dst, src string, excludes map[string]struct{}, errorHandler XAttrErrorHandler) error { return nil } diff --git a/vendor/github.com/containerd/continuity/fs/fstest/file.go b/vendor/github.com/containerd/continuity/fs/fstest/file.go index 574b67576..97e882eaf 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/file.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/file.go @@ -65,7 +65,12 @@ func writeFileStream(name string, stream func() io.Reader, perm os.FileMode) App return err } defer func() { - err := f.Close() + err := f.Sync() + if err != nil && retErr == nil { + retErr = err + } + + err = f.Close() if err != nil && retErr == nil { retErr = err } diff --git a/vendor/modules.txt b/vendor/modules.txt index 10a2c0b25..8c57824c9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -98,7 +98,7 @@ github.com/containerd/cgroups/v3/cgroup2/stats # github.com/containerd/console v1.0.3 ## explicit; go 1.13 github.com/containerd/console -# github.com/containerd/continuity v0.4.0 +# github.com/containerd/continuity v0.4.2-0.20230616210509-1e0d26eb2381 ## explicit; go 1.19 github.com/containerd/continuity github.com/containerd/continuity/devices