From 938f3185bd2df0617352fe44ebf473ad953b02d7 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 10 Jul 2017 11:37:39 -0700 Subject: [PATCH] Add content test suite Add content test suite with test for writer. Update fs and metadata implementations to use test suite. Signed-off-by: Derek McGowan --- content/{content_test.go => fs/store_test.go} | 28 +- content/fs/writer.go | 4 + content/testsuite/testsuite.go | 281 ++++++++++++++++++ metadata/content.go | 3 +- metadata/content_test.go | 30 ++ 5 files changed, 337 insertions(+), 9 deletions(-) rename content/{content_test.go => fs/store_test.go} (84%) create mode 100644 content/testsuite/testsuite.go create mode 100644 metadata/content_test.go diff --git a/content/content_test.go b/content/fs/store_test.go similarity index 84% rename from content/content_test.go rename to content/fs/store_test.go index 3731628d1..7cbdd45e4 100644 --- a/content/content_test.go +++ b/content/fs/store_test.go @@ -1,4 +1,4 @@ -package content +package fs import ( "bufio" @@ -17,10 +17,22 @@ import ( "testing" "time" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/testsuite" "github.com/containerd/containerd/testutil" "github.com/opencontainers/go-digest" ) +func TestContent(t *testing.T) { + testsuite.ContentSuite(t, "fs", func(ctx context.Context, root string) (content.Store, func(), error) { + cs, err := NewStore(root) + if err != nil { + return nil, nil, err + } + return cs, func() {}, nil + }) +} + func TestContentWriter(t *testing.T) { ctx, tmpdir, cs, cleanup := contentStoreEnv(t) defer cleanup() @@ -63,7 +75,7 @@ func TestContentWriter(t *testing.T) { ingestions[i].StartedAt = time.Time{} } - if !reflect.DeepEqual(ingestions, []Status{ + if !reflect.DeepEqual(ingestions, []content.Status{ { Ref: "myref", Offset: 0, @@ -132,7 +144,7 @@ func TestWalkBlobs(t *testing.T) { expected[dgst] = struct{}{} } - if err := cs.Walk(ctx, func(bi Info) error { + if err := cs.Walk(ctx, func(bi content.Info) error { found[bi.Digest] = struct{}{} checkBlobPath(t, cs, bi.Digest) return nil @@ -201,7 +213,7 @@ func generateBlobs(t checker, nblobs, maxsize int64) map[digest.Digest][]byte { return blobs } -func populateBlobStore(t checker, ctx context.Context, cs Store, nblobs, maxsize int64) map[digest.Digest][]byte { +func populateBlobStore(t checker, ctx context.Context, cs content.Store, nblobs, maxsize int64) map[digest.Digest][]byte { blobs := generateBlobs(t, nblobs, maxsize) for dgst, p := range blobs { @@ -211,7 +223,7 @@ func populateBlobStore(t checker, ctx context.Context, cs Store, nblobs, maxsize return blobs } -func contentStoreEnv(t checker) (context.Context, string, Store, func()) { +func contentStoreEnv(t checker) (context.Context, string, content.Store, func()) { pc, _, _, ok := runtime.Caller(1) if !ok { t.Fatal("failed to resolve caller") @@ -247,7 +259,7 @@ func checkCopy(t checker, size int64, dst io.Writer, src io.Reader) { } } -func checkBlobPath(t *testing.T, cs Store, dgst digest.Digest) string { +func checkBlobPath(t *testing.T, cs content.Store, dgst digest.Digest) string { path := cs.(*store).blobPath(dgst) if path != filepath.Join(cs.(*store).root, "blobs", dgst.Algorithm().String(), dgst.Hex()) { @@ -268,8 +280,8 @@ func checkBlobPath(t *testing.T, cs Store, dgst digest.Digest) string { return path } -func checkWrite(t checker, ctx context.Context, cs Store, dgst digest.Digest, p []byte) digest.Digest { - if err := WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p), int64(len(p)), dgst); err != nil { +func checkWrite(t checker, ctx context.Context, cs content.Store, dgst digest.Digest, p []byte) digest.Digest { + if err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p), int64(len(p)), dgst); err != nil { t.Fatal(err) } diff --git a/content/fs/writer.go b/content/fs/writer.go index cfa64f9ea..8b1c76aee 100644 --- a/content/fs/writer.go +++ b/content/fs/writer.go @@ -110,6 +110,10 @@ func (w *writer) Commit(size int64, expected digest.Digest) error { } return err } + commitTime := time.Now() + if err := os.Chtimes(target, commitTime, commitTime); err != nil { + return err + } unlock(w.ref) w.fp = nil diff --git a/content/testsuite/testsuite.go b/content/testsuite/testsuite.go new file mode 100644 index 000000000..d305a4351 --- /dev/null +++ b/content/testsuite/testsuite.go @@ -0,0 +1,281 @@ +package testsuite + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "math/rand" + "os" + "testing" + "time" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/testutil" + digest "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// ContentSuite runs a test suite on the snapshotter given a factory function. +func ContentSuite(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func(), error)) { + t.Run("Writer", makeTest(t, name, storeFn, checkContentStoreWriter)) +} + +func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func(), error), fn func(ctx context.Context, t *testing.T, cs content.Store)) func(t *testing.T) { + return func(t *testing.T) { + ctx := namespaces.WithNamespace(context.Background(), name) + + tmpDir, err := ioutil.TempDir("", "content-suite-"+name+"-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + cs, cleanup, err := storeFn(ctx, tmpDir) + if err != nil { + t.Fatal(err) + } + defer cleanup() + + defer testutil.DumpDir(t, tmpDir) + fn(ctx, t, cs) + } +} + +func checkContentStoreWriter(ctx context.Context, t *testing.T, cs content.Store) { + c1, d1 := createContent(256, 1) + w1, err := cs.Writer(ctx, "c1", 0, "") + if err != nil { + t.Fatal(err) + } + + c2, d2 := createContent(256, 2) + w2, err := cs.Writer(ctx, "c2", int64(len(c2)), "") + if err != nil { + t.Fatal(err) + } + + c3, d3 := createContent(256, 3) + w3, err := cs.Writer(ctx, "c3", 0, d3) + if err != nil { + t.Fatal(err) + } + + c4, d4 := createContent(256, 4) + w4, err := cs.Writer(ctx, "c4", int64(len(c4)), d4) + if err != nil { + t.Fatal(err) + } + + smallbuf := make([]byte, 32) + for _, s := range []struct { + content []byte + digest digest.Digest + writer content.Writer + }{ + { + content: c1, + digest: d1, + writer: w1, + }, + { + content: c2, + digest: d2, + writer: w2, + }, + { + content: c3, + digest: d3, + writer: w3, + }, + { + content: c4, + digest: d4, + writer: w4, + }, + } { + n, err := io.CopyBuffer(s.writer, bytes.NewReader(s.content), smallbuf) + if err != nil { + t.Fatal(err) + } + + if n != int64(len(s.content)) { + t.Fatalf("Unexpected copy length %d, expected %d", n, len(s.content)) + } + + preCommit := time.Now() + if err := s.writer.Commit(0, ""); err != nil { + t.Fatal(err) + } + postCommit := time.Now() + + if s.writer.Digest() != s.digest { + t.Fatalf("Unexpected commit digest %s, expected %s", s.writer.Digest(), s.digest) + } + + info := content.Info{ + Digest: s.digest, + Size: int64(len(s.content)), + } + if err := checkInfo(ctx, cs, s.digest, info, preCommit, postCommit, preCommit, postCommit); err != nil { + t.Fatalf("Check info failed: %+v", err) + } + } +} + +func checkUploadStatus(ctx context.Context, t *testing.T, cs content.Store) { + c1, d1 := createContent(256, 1) + + preStart := time.Now() + w1, err := cs.Writer(ctx, "c1", 256, d1) + if err != nil { + t.Fatal(err) + } + postStart := time.Now() + + d := digest.FromBytes([]byte{}) + + expected := content.Status{ + Ref: "c1", + Total: 256, + Expected: d1, + } + preUpdate := preStart + postUpdate := postStart + + if err := checkStatus(w1, expected, d, preStart, postStart, preUpdate, postUpdate); err != nil { + t.Fatalf("Status check failed: %+v", err) + } + + // Write first 64 bytes + preUpdate = time.Now() + if _, err := w1.Write(c1[:64]); err != nil { + t.Fatalf("Failed to write: %+v", err) + } + postUpdate = time.Now() + expected.Offset = 64 + d = digest.FromBytes(c1[:64]) + if err := checkStatus(w1, expected, d, preStart, postStart, preUpdate, postUpdate); err != nil { + t.Fatalf("Status check failed: %+v", err) + } + + // Write next 128 bytes + preUpdate = time.Now() + if _, err := w1.Write(c1[64:128]); err != nil { + t.Fatalf("Failed to write: %+v", err) + } + postUpdate = time.Now() + expected.Offset = 192 + d = digest.FromBytes(c1[64:128]) + if err := checkStatus(w1, expected, d, preStart, postStart, preUpdate, postUpdate); err != nil { + t.Fatalf("Status check failed: %+v", err) + } + + // Write last 64 bytes + preUpdate = time.Now() + if _, err := w1.Write(c1[192:]); err != nil { + t.Fatalf("Failed to write: %+v", err) + } + postUpdate = time.Now() + expected.Offset = 256 + d = digest.FromBytes(c1[192:]) + if err := checkStatus(w1, expected, d, preStart, postStart, preUpdate, postUpdate); err != nil { + t.Fatalf("Status check failed: %+v", err) + } + + preCommit := time.Now() + if err := w1.Commit(0, ""); err != nil { + t.Fatalf("Commit failed: %+v", err) + } + postCommit := time.Now() + + info := content.Info{ + Digest: d1, + Size: 256, + } + + if err := checkInfo(ctx, cs, d1, info, preCommit, postCommit, preCommit, postCommit); err != nil { + t.Fatalf("Check info failed: %+v", err) + } +} + +func checkStatus(w content.Writer, expected content.Status, d digest.Digest, preStart, postStart, preUpdate, postUpdate time.Time) error { + st, err := w.Status() + if err != nil { + return errors.Wrap(err, "failed to get status") + } + + wd := w.Digest() + if wd != d { + return errors.Errorf("unexpected digest %v, expected %v", wd, d) + } + + if st.Ref != expected.Ref { + return errors.Errorf("unexpected ref %v, expected %v", st.Ref, expected.Ref) + } + + if st.Offset != expected.Offset { + return errors.Errorf("unexpected offset %d, expected %d", st.Offset, expected.Offset) + } + + if st.Total != expected.Total { + return errors.Errorf("unexpected total %d, expected %d", st.Total, expected.Total) + } + + if st.Expected != expected.Expected { + return errors.Errorf("unexpected \"expected digest\" %v, expected %v", st.Expected, expected.Expected) + } + + if st.StartedAt.After(postStart) || st.StartedAt.Before(preStart) { + return errors.Errorf("unexpected started at time %s, expected between %s and %s", st.StartedAt, preStart, postStart) + } + if st.UpdatedAt.After(postUpdate) || st.UpdatedAt.Before(preUpdate) { + return errors.Errorf("unexpected updated at time %s, expected between %s and %s", st.UpdatedAt, preUpdate, postUpdate) + } + + return nil +} + +func checkInfo(ctx context.Context, cs content.Store, d digest.Digest, expected content.Info, c1, c2, u1, u2 time.Time) error { + info, err := cs.Info(ctx, d) + if err != nil { + return errors.Wrap(err, "failed to get info") + } + + if info.Digest != d { + return errors.Errorf("unexpected info digest %s, expected %s", info.Digest, d) + } + + if info.Size != expected.Size { + return errors.Errorf("unexpected info size %d, expected %d", info.Size, expected.Size) + } + + if info.CreatedAt.After(c2) || info.CreatedAt.Before(c1) { + return errors.Errorf("unexpected created at time %s, expected between %s and %s", info.CreatedAt, c1, c2) + } + if info.UpdatedAt.After(u2) || info.UpdatedAt.Before(u1) { + return errors.Errorf("unexpected updated at time %s, expected between %s and %s", info.UpdatedAt, u1, u2) + } + + if len(info.Labels) != len(expected.Labels) { + return errors.Errorf("mismatched number of labels\ngot:\n%#v\nexpected:\n%#v", info.Labels, expected.Labels) + } + + for k, v := range expected.Labels { + actual := info.Labels[k] + if v != actual { + return errors.Errorf("unexpected value for label %q: %q, expected %q", k, actual, v) + } + } + + return nil +} + +func createContent(size, seed int64) ([]byte, digest.Digest) { + b, err := ioutil.ReadAll(io.LimitReader(rand.New(rand.NewSource(seed)), size)) + if err != nil { + panic(err) + } + return b, digest.FromBytes(b) +} diff --git a/metadata/content.go b/metadata/content.go index ceefef15c..a125c8fa3 100644 --- a/metadata/content.go +++ b/metadata/content.go @@ -385,13 +385,14 @@ func (nw *namespacedWriter) commit(tx *bolt.Tx, size int64, expected digest.Dige return err } - timeEncoded, err := status.UpdatedAt.MarshalBinary() + timeEncoded, err := time.Now().UTC().MarshalBinary() if err != nil { return err } for _, v := range [][2][]byte{ {bucketKeyCreatedAt, timeEncoded}, + {bucketKeyUpdatedAt, timeEncoded}, {bucketKeySize, sizeEncoded}, } { if err := bkt.Put(v[0], v[1]); err != nil { diff --git a/metadata/content_test.go b/metadata/content_test.go new file mode 100644 index 000000000..9ce2fed8b --- /dev/null +++ b/metadata/content_test.go @@ -0,0 +1,30 @@ +package metadata + +import ( + "context" + "path/filepath" + "testing" + + "github.com/boltdb/bolt" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/fs" + "github.com/containerd/containerd/content/testsuite" +) + +func TestContent(t *testing.T) { + testsuite.ContentSuite(t, "metadata", func(ctx context.Context, root string) (content.Store, func(), error) { + // TODO: Use mocked or in-memory store + cs, err := fs.NewStore(root) + if err != nil { + return nil, nil, err + } + + db, err := bolt.Open(filepath.Join(root, "metadata.db"), 0660, nil) + if err != nil { + return nil, nil, err + } + cs = NewContentStore(db, cs) + + return cs, func() {}, nil + }) +}