From 5a0ff41c81f4562b8a3dcffc80be6426530c2bb8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 15 Aug 2019 11:03:55 -0700 Subject: [PATCH] Update continuity vendor Pulls in copy and fstest changes Signed-off-by: Derek McGowan --- mount/lookup_linux_test.go | 18 +++--- snapshots/btrfs/btrfs_test.go | 16 +++--- snapshots/overlay/check_test.go | 16 +++--- vendor.conf | 2 +- .../github.com/containerd/continuity/LICENSE | 19 ++----- .../containerd/continuity/README.md | 10 ++++ .../containerd/continuity/fs/copy.go | 51 ++++++++++++++--- .../containerd/continuity/fs/copy_linux.go | 37 +++++++++++-- .../containerd/continuity/fs/copy_unix.go | 24 ++++++-- .../containerd/continuity/fs/copy_windows.go | 2 +- .../continuity/fs/fstest/compare.go | 10 +--- .../continuity/fs/fstest/compare_windows.go | 1 + .../continuity/fs/fstest/continuity_util.go | 12 +++- .../containerd/continuity/fs/path.go | 33 ++++++++++- .../testutil/loopback/loopback_linux.go | 55 +++++++++++++++++-- 15 files changed, 228 insertions(+), 78 deletions(-) diff --git a/mount/lookup_linux_test.go b/mount/lookup_linux_test.go index 7ec9e3ad6..e9347566e 100644 --- a/mount/lookup_linux_test.go +++ b/mount/lookup_linux_test.go @@ -50,25 +50,25 @@ func testLookup(t *testing.T, fsType string) { } defer os.RemoveAll(mnt) - deviceName, cleanupDevice, err := loopback.New(100 << 20) // 100 MB + loop, err := loopback.New(100 << 20) // 100 MB if err != nil { t.Fatal(err) } - if out, err := exec.Command("mkfs", "-t", fsType, deviceName).CombinedOutput(); err != nil { + if out, err := exec.Command("mkfs", "-t", fsType, loop.Device).CombinedOutput(); err != nil { // not fatal - cleanupDevice() - t.Skipf("could not mkfs (%s) %s: %v (out: %q)", fsType, deviceName, err, string(out)) + loop.Close() + t.Skipf("could not mkfs (%s) %s: %v (out: %q)", fsType, loop.Device, err, string(out)) } - if out, err := exec.Command("mount", deviceName, mnt).CombinedOutput(); err != nil { + if out, err := exec.Command("mount", loop.Device, mnt).CombinedOutput(); err != nil { // not fatal - cleanupDevice() - t.Skipf("could not mount %s: %v (out: %q)", deviceName, err, string(out)) + loop.Close() + t.Skipf("could not mount %s: %v (out: %q)", loop.Device, err, string(out)) } defer func() { testutil.Unmount(t, mnt) - cleanupDevice() + loop.Close() }() - assert.Check(t, strings.HasPrefix(deviceName, "/dev/loop")) + assert.Check(t, strings.HasPrefix(loop.Device, "/dev/loop")) checkLookup(t, fsType, mnt, mnt) newMnt, err := ioutil.TempDir("", "containerd-mountinfo-test-newMnt") diff --git a/snapshots/btrfs/btrfs_test.go b/snapshots/btrfs/btrfs_test.go index 6e5102356..842f80d83 100644 --- a/snapshots/btrfs/btrfs_test.go +++ b/snapshots/btrfs/btrfs_test.go @@ -52,24 +52,24 @@ func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshots.Snap if os.Getpagesize() > 4096 { loopbackSize = int64(650 << 20) // 650 MB } - deviceName, cleanupDevice, err := loopback.New(loopbackSize) + loop, err := loopback.New(loopbackSize) if err != nil { return nil, nil, err } - if out, err := exec.Command(mkbtrfs, deviceName).CombinedOutput(); err != nil { - cleanupDevice() + if out, err := exec.Command(mkbtrfs, loop.Device).CombinedOutput(); err != nil { + loop.Close() return nil, nil, errors.Wrapf(err, "failed to make btrfs filesystem (out: %q)", out) } - if out, err := exec.Command("mount", deviceName, root).CombinedOutput(); err != nil { - cleanupDevice() - return nil, nil, errors.Wrapf(err, "failed to mount device %s (out: %q)", deviceName, out) + if out, err := exec.Command("mount", loop.Device, root).CombinedOutput(); err != nil { + loop.Close() + return nil, nil, errors.Wrapf(err, "failed to mount device %s (out: %q)", loop.Device, out) } snapshotter, err := NewSnapshotter(root) if err != nil { - cleanupDevice() + loop.Close() return nil, nil, errors.Wrap(err, "failed to create new snapshotter") } @@ -78,7 +78,7 @@ func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshots.Snap return err } err := mount.UnmountAll(root, unix.MNT_DETACH) - if cerr := cleanupDevice(); cerr != nil { + if cerr := loop.Close(); cerr != nil { err = errors.Wrap(cerr, "device cleanup failed") } return err diff --git a/snapshots/overlay/check_test.go b/snapshots/overlay/check_test.go index 490145e01..16be50e90 100644 --- a/snapshots/overlay/check_test.go +++ b/snapshots/overlay/check_test.go @@ -36,23 +36,23 @@ func testOverlaySupported(t testing.TB, expected bool, mkfs ...string) { } defer os.RemoveAll(mnt) - deviceName, cleanupDevice, err := loopback.New(100 << 20) // 100 MB + loop, err := loopback.New(100 << 20) // 100 MB if err != nil { t.Fatal(err) } - if out, err := exec.Command(mkfs[0], append(mkfs[1:], deviceName)...).CombinedOutput(); err != nil { + if out, err := exec.Command(mkfs[0], append(mkfs[1:], loop.Device)...).CombinedOutput(); err != nil { // not fatal - cleanupDevice() - t.Skipf("could not mkfs (%v) %s: %v (out: %q)", mkfs, deviceName, err, string(out)) + loop.Close() + t.Skipf("could not mkfs (%v) %s: %v (out: %q)", mkfs, loop.Device, err, string(out)) } - if out, err := exec.Command("mount", deviceName, mnt).CombinedOutput(); err != nil { + if out, err := exec.Command("mount", loop.Device, mnt).CombinedOutput(); err != nil { // not fatal - cleanupDevice() - t.Skipf("could not mount %s: %v (out: %q)", deviceName, err, string(out)) + loop.Close() + t.Skipf("could not mount %s: %v (out: %q)", loop.Device, err, string(out)) } defer func() { testutil.Unmount(t, mnt) - cleanupDevice() + loop.Close() }() workload := func() { err = Supported(mnt) diff --git a/vendor.conf b/vendor.conf index edb2f4ea3..95379974a 100644 --- a/vendor.conf +++ b/vendor.conf @@ -4,7 +4,7 @@ github.com/containerd/cgroups c4b9ac5c7601384c965b9646fc515884e091ebb9 github.com/containerd/typeurl a93fcdb778cd272c6e9b3028b2f42d813e785d40 github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c github.com/containerd/btrfs af5082808c833de0e79c1e72eea9fea239364877 -github.com/containerd/continuity bd77b46c8352f74eb12c85bdc01f4b90f69d66b4 +github.com/containerd/continuity f2a389ac0a02ce21c09edd7344677a601970f41c github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 github.com/docker/go-metrics 4ea375f7759c82740c893fc030bc37088d2ec098 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 diff --git a/vendor/github.com/containerd/continuity/LICENSE b/vendor/github.com/containerd/continuity/LICENSE index 8f71f43fe..584149b6e 100644 --- a/vendor/github.com/containerd/continuity/LICENSE +++ b/vendor/github.com/containerd/continuity/LICENSE @@ -1,6 +1,7 @@ + Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -175,28 +176,16 @@ END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} + Copyright The containerd Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/vendor/github.com/containerd/continuity/README.md b/vendor/github.com/containerd/continuity/README.md index 0e91ce07b..f9f9ef0f9 100644 --- a/vendor/github.com/containerd/continuity/README.md +++ b/vendor/github.com/containerd/continuity/README.md @@ -72,3 +72,13 @@ If you change the proto file you will need to rebuild the generated Go with `go ```console $ go generate ./proto ``` + +## Project details + +continuity is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE). +As a containerd sub-project, you will find the: + * [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md), + * [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS), + * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md) + +information in our [`containerd/project`](https://github.com/containerd/project) repository. diff --git a/vendor/github.com/containerd/continuity/fs/copy.go b/vendor/github.com/containerd/continuity/fs/copy.go index 42df6a9a5..ad61022ad 100644 --- a/vendor/github.com/containerd/continuity/fs/copy.go +++ b/vendor/github.com/containerd/continuity/fs/copy.go @@ -32,14 +32,49 @@ var bufferPool = &sync.Pool{ }, } -// CopyDir copies the directory from src to dst. -// Most efficient copy of files is attempted. -func CopyDir(dst, src string) error { - inodes := map[uint64]string{} - return copyDirectory(dst, src, inodes) +// XAttrErrorHandlers transform a non-nil xattr error. +// Return nil to ignore an error. +// xattrKey can be empty for listxattr operation. +type XAttrErrorHandler func(dst, src, xattrKey string, err error) error + +type copyDirOpts struct { + xeh XAttrErrorHandler } -func copyDirectory(dst, src string, inodes map[uint64]string) error { +type CopyDirOpt func(*copyDirOpts) error + +// WithXAttrErrorHandler allows specifying XAttrErrorHandler +// If nil XAttrErrorHandler is specified (default), CopyDir stops +// on a non-nil xattr error. +func WithXAttrErrorHandler(xeh XAttrErrorHandler) CopyDirOpt { + return func(o *copyDirOpts) error { + o.xeh = xeh + return nil + } +} + +// WithAllowXAttrErrors allows ignoring xattr errors. +func WithAllowXAttrErrors() CopyDirOpt { + xeh := func(dst, src, xattrKey string, err error) error { + return nil + } + return WithXAttrErrorHandler(xeh) +} + +// CopyDir copies the directory from src to dst. +// Most efficient copy of files is attempted. +func CopyDir(dst, src string, opts ...CopyDirOpt) error { + var o copyDirOpts + for _, opt := range opts { + if err := opt(&o); err != nil { + return err + } + } + inodes := map[uint64]string{} + return copyDirectory(dst, src, inodes, &o) +} + +func copyDirectory(dst, src string, inodes map[uint64]string, o *copyDirOpts) error { stat, err := os.Stat(src) if err != nil { return errors.Wrapf(err, "failed to stat %s", src) @@ -75,7 +110,7 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { switch { case fi.IsDir(): - if err := copyDirectory(target, source, inodes); err != nil { + if err := copyDirectory(target, source, inodes, o); err != nil { return err } continue @@ -111,7 +146,7 @@ func copyDirectory(dst, src string, inodes map[uint64]string) error { return errors.Wrap(err, "failed to copy file info") } - if err := copyXAttrs(target, source); err != nil { + if err := copyXAttrs(target, source, o.xeh); err != nil { return errors.Wrap(err, "failed to copy xattrs") } } diff --git a/vendor/github.com/containerd/continuity/fs/copy_linux.go b/vendor/github.com/containerd/continuity/fs/copy_linux.go index e041b5661..81c71522a 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_linux.go +++ b/vendor/github.com/containerd/continuity/fs/copy_linux.go @@ -59,6 +59,8 @@ func copyFileInfo(fi os.FileInfo, name string) error { return nil } +const maxSSizeT = int64(^uint(0) >> 1) + func copyFileContent(dst, src *os.File) error { st, err := src.Stat() if err != nil { @@ -71,7 +73,16 @@ func copyFileContent(dst, src *os.File) error { dstFd := int(dst.Fd()) for size > 0 { - n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, int(size), 0) + // Ensure that we are never trying to copy more than SSIZE_MAX at a + // time and at the same time avoids overflows when the file is larger + // than 4GB on 32-bit systems. + var copySize int + if size > maxSSizeT { + copySize = int(maxSSizeT) + } else { + copySize = int(size) + } + n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, copySize, 0) if err != nil { if (err != unix.ENOSYS && err != unix.EXDEV) || !first { return errors.Wrap(err, "copy file range failed") @@ -90,18 +101,34 @@ func copyFileContent(dst, src *os.File) error { return nil } -func copyXAttrs(dst, src string) error { +func copyXAttrs(dst, src string, xeh XAttrErrorHandler) error { xattrKeys, err := sysx.LListxattr(src) if err != nil { - return errors.Wrapf(err, "failed to list xattrs on %s", src) + e := errors.Wrapf(err, "failed to list xattrs on %s", src) + if xeh != nil { + e = xeh(dst, src, "", e) + } + return e } for _, xattr := range xattrKeys { data, err := sysx.LGetxattr(src, xattr) if err != nil { - return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + e := errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + if xeh != nil { + if e = xeh(dst, src, xattr, e); e == nil { + continue + } + } + return e } if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { - return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + e := errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + if xeh != nil { + if e = xeh(dst, src, xattr, e); e == nil { + continue + } + } + return e } } diff --git a/vendor/github.com/containerd/continuity/fs/copy_unix.go b/vendor/github.com/containerd/continuity/fs/copy_unix.go index 1a8ae5ebd..73c01a46d 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_unix.go +++ b/vendor/github.com/containerd/continuity/fs/copy_unix.go @@ -69,18 +69,34 @@ func copyFileContent(dst, src *os.File) error { return err } -func copyXAttrs(dst, src string) error { +func copyXAttrs(dst, src string, xeh XAttrErrorHandler) error { xattrKeys, err := sysx.LListxattr(src) if err != nil { - return errors.Wrapf(err, "failed to list xattrs on %s", src) + e := errors.Wrapf(err, "failed to list xattrs on %s", src) + if xeh != nil { + e = xeh(dst, src, "", e) + } + return e } for _, xattr := range xattrKeys { data, err := sysx.LGetxattr(src, xattr) if err != nil { - return errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + e := errors.Wrapf(err, "failed to get xattr %q on %s", xattr, src) + if xeh != nil { + if e = xeh(dst, src, xattr, e); e == nil { + continue + } + } + return e } if err := sysx.LSetxattr(dst, xattr, data, 0); err != nil { - return errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + e := errors.Wrapf(err, "failed to set xattr %q on %s", xattr, dst) + if xeh != nil { + if e = xeh(dst, src, xattr, e); e == nil { + continue + } + } + return e } } diff --git a/vendor/github.com/containerd/continuity/fs/copy_windows.go b/vendor/github.com/containerd/continuity/fs/copy_windows.go index be8e6489b..27c7d7dbb 100644 --- a/vendor/github.com/containerd/continuity/fs/copy_windows.go +++ b/vendor/github.com/containerd/continuity/fs/copy_windows.go @@ -40,7 +40,7 @@ func copyFileContent(dst, src *os.File) error { return err } -func copyXAttrs(dst, src string) error { +func copyXAttrs(dst, src string, xeh XAttrErrorHandler) error { return nil } diff --git a/vendor/github.com/containerd/continuity/fs/fstest/compare.go b/vendor/github.com/containerd/continuity/fs/fstest/compare.go index b61d83082..0d100b624 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/compare.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/compare.go @@ -49,15 +49,7 @@ func CheckDirectoryEqual(d1, d2 string) error { diff := diffResourceList(m1.Resources, m2.Resources) if diff.HasDiff() { - if len(diff.Deletions) != 0 { - return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) - } - // TODO: Also skip Recycle Bin contents in Windows layers which is used to store deleted files in some cases - for _, add := range diff.Additions { - if ok, _ := metadataFiles[add.Path()]; !ok { - return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) - } - } + return errors.Errorf("directory diff between %s and %s\n%s", d1, d2, diff.String()) } return nil diff --git a/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go b/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go index 6b9104de0..a35781999 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/compare_windows.go @@ -17,6 +17,7 @@ package fstest // TODO: Any more metadata files generated by Windows layers? +// TODO: Also skip Recycle Bin contents in Windows layers which is used to store deleted files in some cases var metadataFiles = map[string]bool{ "\\System Volume Information": true, "\\WcSandboxState": true, diff --git a/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go b/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go index 9cbfc0b6c..4d30dd01f 100644 --- a/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go +++ b/vendor/github.com/containerd/continuity/fs/fstest/continuity_util.go @@ -42,7 +42,17 @@ type resourceListDifference struct { } func (l resourceListDifference) HasDiff() bool { - return len(l.Additions) > 0 || len(l.Deletions) > 0 || len(l.Updates) > 0 + if len(l.Deletions) > 0 || len(l.Updates) > 0 || (len(metadataFiles) == 0 && len(l.Additions) > 0) { + return true + } + + for _, add := range l.Additions { + if ok, _ := metadataFiles[add.Path()]; !ok { + return true + } + } + + return false } func (l resourceListDifference) String() string { diff --git a/vendor/github.com/containerd/continuity/fs/path.go b/vendor/github.com/containerd/continuity/fs/path.go index 995981780..8863caa9d 100644 --- a/vendor/github.com/containerd/continuity/fs/path.go +++ b/vendor/github.com/containerd/continuity/fs/path.go @@ -22,7 +22,6 @@ import ( "io" "os" "path/filepath" - "strings" "github.com/pkg/errors" ) @@ -47,9 +46,8 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { if upper == nil { return ChangeKindDelete, lower.path } - // TODO: compare by directory - switch i := strings.Compare(lower.path, upper.path); { + switch i := directoryCompare(lower.path, upper.path); { case i < 0: // File in lower that is not in upper return ChangeKindDelete, lower.path @@ -61,6 +59,35 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { } } +func directoryCompare(a, b string) int { + l := len(a) + if len(b) < l { + l = len(b) + } + for i := 0; i < l; i++ { + c1, c2 := a[i], b[i] + if c1 == filepath.Separator { + c1 = byte(0) + } + if c2 == filepath.Separator { + c2 = byte(0) + } + if c1 < c2 { + return -1 + } + if c1 > c2 { + return +1 + } + } + if len(a) < len(b) { + return -1 + } + if len(a) > len(b) { + return +1 + } + return 0 +} + func sameFile(f1, f2 *currentPath) (bool, error) { if os.SameFile(f1.f, f2.f) { return true, nil diff --git a/vendor/github.com/containerd/continuity/testutil/loopback/loopback_linux.go b/vendor/github.com/containerd/continuity/testutil/loopback/loopback_linux.go index b17361170..c404c2699 100644 --- a/vendor/github.com/containerd/continuity/testutil/loopback/loopback_linux.go +++ b/vendor/github.com/containerd/continuity/testutil/loopback/loopback_linux.go @@ -22,24 +22,25 @@ import ( "io/ioutil" "os" "os/exec" + "syscall" "strings" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) -// New creates a loopback device, and returns its device name (/dev/loopX), and its clean-up function. -func New(size int64) (string, func() error, error) { +// New creates a loopback device +func New(size int64) (*Loopback, error) { // create temporary file for the disk image file, err := ioutil.TempFile("", "containerd-test-loopback") if err != nil { - return "", nil, errors.Wrap(err, "could not create temporary file for loopback") + return nil, errors.Wrap(err, "could not create temporary file for loopback") } if err := file.Truncate(size); err != nil { file.Close() os.Remove(file.Name()) - return "", nil, errors.Wrap(err, "failed to resize temp file") + return nil, errors.Wrap(err, "failed to resize temp file") } file.Close() @@ -48,7 +49,7 @@ func New(size int64) (string, func() error, error) { p, err := losetup.Output() if err != nil { os.Remove(file.Name()) - return "", nil, errors.Wrap(err, "loopback setup failed") + return nil, errors.Wrap(err, "loopback setup failed") } deviceName := strings.TrimSpace(string(p)) @@ -68,5 +69,47 @@ func New(size int64) (string, func() error, error) { return os.Remove(file.Name()) } - return deviceName, cleanup, nil + l := Loopback{ + File: file.Name(), + Device: deviceName, + close: cleanup, + } + return &l, nil +} + +// Loopback device +type Loopback struct { + // File is the underlying sparse file + File string + // Device is /dev/loopX + Device string + close func() error +} + +// SoftSize returns st_size +func (l *Loopback) SoftSize() (int64, error) { + st, err := os.Stat(l.File) + if err != nil { + return 0, err + } + return st.Size(), nil +} + +// HardSize returns st_blocks * 512; see stat(2) +func (l *Loopback) HardSize() (int64, error) { + st, err := os.Stat(l.File) + if err != nil { + return 0, err + } + st2, ok := st.Sys().(*syscall.Stat_t) + if !ok { + return 0, errors.New("st.Sys() is not a *syscall.Stat_t") + } + // NOTE: st_blocks has nothing to do with st_blksize; see stat(2) + return st2.Blocks * 512, nil +} + +// Close detaches the device and removes the underlying file +func (l *Loopback) Close() error { + return l.close() }