github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #1414 from dmcgowan/btrfs-test-fail-2

Browse Source 
More snapshot and btrfs test cleanup
This commit is contained in:
Michael Crosby 2017-08-23 09:44:05 -04:00 committed by GitHub
commit 311ea33608
16 changed files with 117 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
fs/dtype_linux_test.go
View File

@ -20,7 +20,10 @@ func testSupportsDType(t *testing.T, expected bool, mkfs ...string) {
}
defer os.RemoveAll(mnt)
deviceName, cleanupDevice := testutil.NewLoopback(t, 100<<20) // 100 MB
deviceName, cleanupDevice, err := testutil.NewLoopback(100 << 20) // 100 MB
if err != nil {
t.Fatal(err)
}
if out, err := exec.Command(mkfs[0], append(mkfs[1:], deviceName)...).CombinedOutput(); err != nil {
// not fatal
t.Skipf("could not mkfs (%v) %s: %v (out: %q)", mkfs, deviceName, err, string(out))

6
metadata/snapshot_test.go
View File

@ -13,7 +13,7 @@ import (
"github.com/containerd/containerd/testutil"
)
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error) {
naiveRoot := filepath.Join(root, "naive")
if err := os.Mkdir(naiveRoot, 0770); err != nil {
return nil, nil, err
@ -30,8 +30,8 @@ func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, fun
sn := NewSnapshotter(db, "naive", snapshotter)
return sn, func() {
db.Close()
return sn, func() error {
return db.Close()
}, nil
}

5
mount/lookup_test/lookup_linux_test.go
View File

@ -29,7 +29,10 @@ func testLookup(t *testing.T, fsType string) {
}
defer os.RemoveAll(mnt)
deviceName, cleanupDevice := testutil.NewLoopback(t, 100<<20) // 100 MB
deviceName, cleanupDevice, err := testutil.NewLoopback(100 << 20) // 100 MB
if err != nil {
t.Fatal(err)
}
if out, err := exec.Command("mkfs", "-t", fsType, deviceName).CombinedOutput(); err != nil {
// not fatal
t.Skipf("could not mkfs (%s) %s: %v (out: %q)", fsType, deviceName, err, string(out))

39
snapshot/btrfs/btrfs_test.go
View File

@ -15,30 +15,45 @@ import (
"github.com/containerd/containerd/snapshot"
"github.com/containerd/containerd/snapshot/testsuite"
"github.com/containerd/containerd/testutil"
"github.com/pkg/errors"
"golang.org/x/sys/unix"
)
func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshot.Snapshotter, func(), error) {
return func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
func boltSnapshotter(t *testing.T) func(context.Context, string) (snapshot.Snapshotter, func() error, error) {
mkbtrfs, err := exec.LookPath("mkfs.btrfs")
if err != nil {
t.Skipf("could not find mkfs.btrfs: %v", err)
}
deviceName, cleanupDevice := testutil.NewLoopback(t, 100<<20) // 100 MB
// TODO: Check for btrfs in /proc/module and skip if not loaded
if out, err := exec.Command("mkfs.btrfs", deviceName).CombinedOutput(); err != nil {
// not fatal
t.Skipf("could not mkfs.btrfs %s: %v (out: %q)", deviceName, err, string(out))
return func(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error) {
deviceName, cleanupDevice, err := testutil.NewLoopback(100 << 20) // 100 MB
if err != nil {
return nil, nil, err
}
if out, err := exec.Command(mkbtrfs, deviceName).CombinedOutput(); err != nil {
return nil, nil, errors.Wrapf(err, "failed to make btrfs filesystem (out: %q)", out)
}
if out, err := exec.Command("mount", deviceName, root).CombinedOutput(); err != nil {
// not fatal
t.Skipf("could not mount %s: %v (out: %q)", deviceName, err, string(out))
return nil, nil, errors.Wrapf(err, "failed to mount device %s (out: %q)", deviceName, out)
}
snapshotter, err := NewSnapshotter(root)
if err != nil {
t.Fatal(err)
return nil, nil, errors.Wrap(err, "failed to create new snapshotter")
}
return snapshotter, func() {
testutil.Unmount(t, root)
cleanupDevice()
return snapshotter, func() (err error) {
merr := mount.UnmountAll(root, unix.MNT_DETACH)
if err = cleanupDevice(); err != nil {
return errors.Wrap(err, "device cleanup failed")
} else {
err = merr
}
return err
}, nil
}
}

4
snapshot/naive/naive_test.go
View File

@ -9,13 +9,13 @@ import (
"github.com/containerd/containerd/testutil"
)
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error) {
snapshotter, err := NewSnapshotter(root)
if err != nil {
return nil, nil, err
}
return snapshotter, func() {}, nil
return snapshotter, nil, nil
}
func TestNaive(t *testing.T) {

4
snapshot/overlay/overlay_test.go
View File

@ -18,13 +18,13 @@ import (
"github.com/containerd/containerd/testutil"
)
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error) {
snapshotter, err := NewSnapshotter(root)
if err != nil {
return nil, nil, err
}
return snapshotter, func() {}, nil
return snapshotter, nil, nil
}
func TestOverlay(t *testing.T) {

24
snapshot/testsuite/helpers.go
View File

@ -21,11 +21,11 @@ func applyToMounts(m []mount.Mount, work string, a fstest.Applier) (err error) {
defer os.RemoveAll(td)
if err := mount.MountAll(m, td); err != nil {
return err
return errors.Wrap(err, "failed to mount")
}
defer func() {
if err1 := mount.UnmountAll(td, 0); err == nil {
err = err1
if err1 := mount.UnmountAll(td, umountflags); err == nil {
err = errors.Wrap(err1, "failed to unmount")
}
}()
@ -40,26 +40,30 @@ func createSnapshot(ctx context.Context, sn snapshot.Snapshotter, parent, work s
m, err := sn.Prepare(ctx, prepare, parent)
if err != nil {
return "", err
return "", errors.Wrap(err, "failed to prepare snapshot")
}
if err := applyToMounts(m, work, a); err != nil {
return "", err
return "", errors.Wrap(err, "failed to apply")
}
if err := sn.Commit(ctx, n, prepare); err != nil {
return "", err
return "", errors.Wrap(err, "failed to commit")
}
return n, nil
}
func checkSnapshot(ctx context.Context, sn snapshot.Snapshotter, work, name, check string) error {
func checkSnapshot(ctx context.Context, sn snapshot.Snapshotter, work, name, check string) (err error) {
td, err := ioutil.TempDir(work, "check")
if err != nil {
return errors.Wrap(err, "failed to create temp dir")
}
defer os.RemoveAll(td)
defer func() {
if err1 := os.RemoveAll(td); err == nil {
err = errors.Wrapf(err1, "failed to remove temporary directory %s", td)
}
}()
view := fmt.Sprintf("%s-view", name)
m, err := sn.View(ctx, view, name)
@ -73,10 +77,10 @@ func checkSnapshot(ctx context.Context, sn snapshot.Snapshotter, work, name, che
}()
if err := mount.MountAll(m, td); err != nil {
return errors.Wrap(err, "failed to unmount")
return errors.Wrap(err, "failed to mount")
}
defer func() {
if err1 := mount.UnmountAll(td, 0); err == nil {
if err1 := mount.UnmountAll(td, umountflags); err == nil {
err = errors.Wrap(err1, "failed to unmount view")
}
}()

5
snapshot/testsuite/helpers_linux.go Normal file
View File

@ -0,0 +1,5 @@
package testsuite
import "golang.org/x/sys/unix"
const umountflags int = unix.MNT_DETACH

5
snapshot/testsuite/helpers_other.go Normal file
View File

@ -0,0 +1,5 @@
// +build !linux
package testsuite
const umountflags int = 0

8
snapshot/testsuite/issues.go
View File

@ -22,8 +22,6 @@ import (
// avoid such issues by not relying on tar to create layers.
// See https://github.com/docker/docker/issues/21555
func checkLayerFileUpdate(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
t.Parallel()
l1Init := fstest.Apply(
fstest.CreateDir("/etc", 0700),
fstest.CreateFile("/etc/hosts", []byte("mydomain 10.0.0.1"), 0644),
@ -55,8 +53,6 @@ func checkLayerFileUpdate(ctx context.Context, t *testing.T, sn snapshot.Snapsho
// checkRemoveDirectoryInLowerLayer
// See https://github.com/docker/docker/issues/25244
func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
t.Parallel()
l1Init := fstest.Apply(
fstest.CreateDir("/lib", 0700),
fstest.CreateFile("/lib/hidden", []byte{}, 0644),
@ -80,8 +76,6 @@ func checkRemoveDirectoryInLowerLayer(ctx context.Context, t *testing.T, sn snap
// See https://github.com/docker/docker/issues/24913 overlay
// see https://github.com/docker/docker/issues/28391 overlay2
func checkChown(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
t.Parallel()
l1Init := fstest.Apply(
fstest.CreateDir("/opt", 0700),
fstest.CreateDir("/opt/a", 0700),
@ -124,8 +118,6 @@ func checkRename(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, wor
// checkDirectoryPermissionOnCommit
// https://github.com/docker/docker/issues/27298
func checkDirectoryPermissionOnCommit(ctx context.Context, t *testing.T, sn snapshot.Snapshotter, work string) {
t.Parallel()
l1Init := fstest.Apply(
fstest.CreateDir("/dir1", 0700),
fstest.CreateDir("/dir2", 0700),

45
snapshot/testsuite/testsuite.go
View File

@ -19,8 +19,9 @@ import (
)
// SnapshotterSuite runs a test suite on the snapshotter given a factory function.
func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error)) {
t.Parallel()
func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error)) {
restoreMask := clearMask()
defer restoreMask()
t.Run("Basic", makeTest(name, snapshotterFn, checkSnapshotterBasic))
t.Run("StatActive", makeTest(name, snapshotterFn, checkSnapshotterStatActive))
@ -39,12 +40,12 @@ func SnapshotterSuite(t *testing.T, name string, snapshotterFn func(ctx context.
//t.Run("Rename", makeTest(name, snapshotterFn, checkRename))
}
func makeTest(name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func(), error), fn func(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string)) func(t *testing.T) {
func makeTest(name string, snapshotterFn func(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error), fn func(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string)) func(t *testing.T) {
return func(t *testing.T) {
t.Parallel()
ctx := context.Background()
ctx = namespaces.WithNamespace(ctx, "testsuite")
restoreMask := clearMask()
defer restoreMask()
// Make two directories: a snapshotter root and a play area for the tests:
//
// /tmp
@ -64,9 +65,15 @@ func makeTest(name string, snapshotterFn func(ctx context.Context, root string)
snapshotter, cleanup, err := snapshotterFn(ctx, root)
if err != nil {
t.Fatal(err)
t.Fatalf("Failed to initialize snapshotter: %+v", err)
}
defer cleanup()
defer func() {
if cleanup != nil {
if err := cleanup(); err != nil {
t.Errorf("Cleanup failed: %v", err)
}
}
}()
work := filepath.Join(tmpDir, "work")
if err := os.MkdirAll(work, 0777); err != nil {
@ -80,9 +87,6 @@ func makeTest(name string, snapshotterFn func(ctx context.Context, root string)
// checkSnapshotterBasic tests the basic workflow of a snapshot snapshotter.
func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
// TODO: this always fails when run in parallel, why?
// t.Parallel()
initialApplier := fstest.Apply(
fstest.CreateFile("/foo", []byte("foo\n"), 0777),
fstest.CreateDir("/a", 0755),
@ -186,7 +190,14 @@ func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapsh
return nil
}))
assert.Equal(t, expected, walked)
for ek, ev := range expected {
av, ok := walked[ek]
if !ok {
t.Errorf("Missing stat for %v", ek)
continue
}
assert.Equal(t, ev, av)
}
nextnext := filepath.Join(work, "nextnextlayer")
if err := os.MkdirAll(nextnext, 0777); err != nil {
@ -216,8 +227,6 @@ func checkSnapshotterBasic(ctx context.Context, t *testing.T, snapshotter snapsh
// Create a New Layer on top of base layer with Prepare, Stat on new layer, should return Active layer.
func checkSnapshotterStatActive(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
preparing := filepath.Join(work, "preparing")
if err := os.MkdirAll(preparing, 0777); err != nil {
t.Fatal(err)
@ -252,8 +261,6 @@ func checkSnapshotterStatActive(ctx context.Context, t *testing.T, snapshotter s
// Commit a New Layer on top of base layer with Prepare & Commit , Stat on new layer, should return Committed layer.
func checkSnapshotterStatCommitted(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
preparing := filepath.Join(work, "preparing")
if err := os.MkdirAll(preparing, 0777); err != nil {
t.Fatal(err)
@ -315,8 +322,6 @@ func snapshotterPrepareMount(ctx context.Context, snapshotter snapshot.Snapshott
// Given A <- B <- C, B is the parent of C and A is a transitive parent of C (in this case, a "grandparent")
func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
preparing, err := snapshotterPrepareMount(ctx, snapshotter, "preparing", "", work)
if err != nil {
t.Fatal(err)
@ -371,8 +376,6 @@ func checkSnapshotterTransitivity(ctx context.Context, t *testing.T, snapshotter
// Creating two layers with Prepare or View with same key must fail.
func checkSnapshotterPrepareView(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
preparing, err := snapshotterPrepareMount(ctx, snapshotter, "preparing", "", work)
if err != nil {
t.Fatal(err)
@ -469,8 +472,6 @@ func baseTestSnapshots(ctx context.Context, snapshotter snapshot.Snapshotter) er
}
func checkUpdate(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
t1 := time.Now().UTC()
if err := baseTestSnapshots(ctx, snapshotter); err != nil {
t.Fatalf("Failed to create base snapshots: %v", err)
@ -622,8 +623,6 @@ func assertLabels(t *testing.T, actual, expected map[string]string) {
}
func checkRemove(ctx context.Context, t *testing.T, snapshotter snapshot.Snapshotter, work string) {
t.Parallel()
if _, err := snapshotter.Prepare(ctx, "committed-a", ""); err != nil {
t.Fatal(err)
}

6
snapshot_test.go
View File

@ -9,7 +9,7 @@ import (
"github.com/containerd/containerd/snapshot/testsuite"
)
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func(), error) {
func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, func() error, error) {
client, err := New(address)
if err != nil {
return nil, nil, err
@ -17,8 +17,8 @@ func newSnapshotter(ctx context.Context, root string) (snapshot.Snapshotter, fun
sn := client.SnapshotService(DefaultSnapshotter)
return sn, func() {
client.Close()
return sn, func() error {
return client.Close()
}, nil
}

2
testutil/helpers_unix.go
View File

@ -14,7 +14,7 @@ import (
// Unmount unmounts a given mountPoint and sets t.Error if it fails
func Unmount(t *testing.T, mountPoint string) {
t.Log("unmount", mountPoint)
if err := mount.Unmount(mountPoint, 0); err != nil {
if err := mount.UnmountAll(mountPoint, umountflags); err != nil {
t.Error("Could not umount", mountPoint, err)
}
}

27
testutil/loopback_linux.go
View File

@ -7,19 +7,21 @@ import (
"os"
"os/exec"
"strings"
"testing"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
// NewLoopback creates a loopback device, and returns its device name (/dev/loopX), and its clean-up function.
func NewLoopback(t *testing.T, size int64) (string, func()) {
func NewLoopback(size int64) (string, func() error, error) {
// create temporary file for the disk image
file, err := ioutil.TempFile("", "containerd-test-loopback")
if err != nil {
t.Fatalf("could not create temporary file for loopback: %v", err)
return "", nil, errors.Wrap(err, "could not create temporary file for loopback")
}
if err := file.Truncate(size); err != nil {
t.Fatal(err)
return "", nil, errors.Wrap(err, "failed to resize temp file")
}
file.Close()
@ -27,27 +29,28 @@ func NewLoopback(t *testing.T, size int64) (string, func()) {
losetup := exec.Command("losetup", "--find", "--show", file.Name())
p, err := losetup.Output()
if err != nil {
t.Fatal(err)
return "", nil, errors.Wrap(err, "loopback setup failed")
}
deviceName := strings.TrimSpace(string(p))
t.Logf("Created loop device %s (using %s)", deviceName, file.Name())
logrus.Debugf("Created loop device %s (using %s)", deviceName, file.Name())
cleanup := func() {
cleanup := func() error {
// detach device
t.Logf("Removing loop device %s", deviceName)
logrus.Debugf("Removing loop device %s", deviceName)
losetup := exec.Command("losetup", "--detach", deviceName)
err := losetup.Run()
if err != nil {
t.Error("Could not remove loop device", deviceName, err)
return errors.Wrapf(err, "Could not remove loop device %s", deviceName)
}
// remove file
t.Logf("Removing temporary file %s", file.Name())
logrus.Debugf("Removing temporary file %s", file.Name())
if err = os.Remove(file.Name()); err != nil {
t.Error(err)
return err
}
return nil
}
return deviceName, cleanup
return deviceName, cleanup, nil
}

5
testutil/mount_linux.go Normal file
View File

@ -0,0 +1,5 @@
package testutil
import "golang.org/x/sys/unix"
const umountflags int = unix.MNT_DETACH

5
testutil/mount_other.go Normal file
View File

@ -0,0 +1,5 @@
// +build !linux
package testutil
const umountflags int = 0