Merge pull request #2154 from dmcgowan/shared-content-ingests

content: shared content across namespaces

content/helpers.go

@@ -78,7 +78,7 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, size i
 }
 
 // Copy copies data with the expected digest from the reader into the
-// provided content store writer.
+// provided content store writer. This copy commits the writer.
 //
 // This is useful when the digest and size are known beforehand. When
 // the size or digest is unknown, these values may be empty.
@@ -113,6 +113,22 @@ func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected dige
 	return nil
 }
 
+// CopyReaderAt copies to a writer from a given reader at for the given
+// number of bytes. This copy does not commit the writer.
+func CopyReaderAt(cw Writer, ra ReaderAt, n int64) error {
+	ws, err := cw.Status()
+	if err != nil {
+		return err
+	}
+
+	buf := bufPool.Get().(*[]byte)
+	defer bufPool.Put(buf)
+
+	_, err = io.CopyBuffer(cw, io.NewSectionReader(ra, ws.Offset, n), *buf)
+
+	return err
+}
+
 // seekReader attempts to seek the reader to the given offset, either by
 // resolving `io.Seeker`, by detecting `io.ReaderAt`, or discarding
 // up to the given offset.

content/testsuite/testsuite.go

@@ -30,7 +30,6 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/testutil"
 	"github.com/gotestyourself/gotestyourself/assert"
 	digest "github.com/opencontainers/go-digest"
@@ -48,11 +47,38 @@ func ContentSuite(t *testing.T, name string, storeFn func(ctx context.Context, r
 	t.Run("ResumeCopySeeker", makeTest(t, name, storeFn, checkResume(resumeCopySeeker)))
 	t.Run("ResumeCopyReaderAt", makeTest(t, name, storeFn, checkResume(resumeCopyReaderAt)))
 	t.Run("Labels", makeTest(t, name, storeFn, checkLabels))
+
+	t.Run("CrossNamespaceAppend", makeTest(t, name, storeFn, checkCrossNSAppend))
+	t.Run("CrossNamespaceShare", makeTest(t, name, storeFn, checkCrossNSShare))
+}
+
+// ContextWrapper is used to decorate new context used inside the test
+// before using the context on the content store.
+// This can be used to support leasing and multiple namespaces tests.
+type ContextWrapper func(ctx context.Context) (context.Context, func() error, error)
+
+type wrapperKey struct{}
+
+// SetContextWrapper sets the wrapper on the context for deriving
+// new test contexts from the context.
+func SetContextWrapper(ctx context.Context, w ContextWrapper) context.Context {
+	return context.WithValue(ctx, wrapperKey{}, w)
+}
+
+type nameKey struct{}
+
+// Name gets the test name from the context
+func Name(ctx context.Context) string {
+	name, ok := ctx.Value(nameKey{}).(string)
+	if !ok {
+		return ""
+	}
+	return name
 }
 
 func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root string) (context.Context, content.Store, func() error, error), fn func(ctx context.Context, t *testing.T, cs content.Store)) func(t *testing.T) {
 	return func(t *testing.T) {
-		ctx := namespaces.WithNamespace(context.Background(), name)
+		ctx := context.WithValue(context.Background(), nameKey{}, name)
 
 		tmpDir, err := ioutil.TempDir("", "content-suite-"+name+"-")
 		if err != nil {
@@ -70,6 +96,20 @@ func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root
 			}
 		}()
 
+		w, ok := ctx.Value(wrapperKey{}).(ContextWrapper)
+		if ok {
+			var done func() error
+			ctx, done, err = w(ctx)
+			if err != nil {
+				t.Fatalf("Error wrapping context: %+v", err)
+			}
+			defer func() {
+				if err := done(); err != nil && !t.Failed() {
+					t.Fatalf("Wrapper release failed: %+v", err)
+				}
+			}()
+		}
+
 		defer testutil.DumpDirOnFailure(t, tmpDir)
 		fn(ctx, t, cs)
 	}
@@ -483,6 +523,124 @@ func resumeCopyReaderAt(ctx context.Context, w content.Writer, b []byte, _, size
 	return errors.Wrap(content.Copy(ctx, w, r, size, dgst), "copy failed")
 }
 
+func checkCrossNSShare(ctx context.Context, t *testing.T, cs content.Store) {
+	wrap, ok := ctx.Value(wrapperKey{}).(ContextWrapper)
+	if !ok {
+		t.Skip("multiple contexts not supported")
+	}
+
+	var size int64 = 1000
+	b, d := createContent(size)
+	ref := fmt.Sprintf("ref-%d", size)
+	t1 := time.Now()
+
+	if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), size, d); err != nil {
+		t.Fatal(err)
+	}
+
+	ctx2, done, err := wrap(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	w, err := cs.Writer(ctx2, ref, size, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t2 := time.Now()
+
+	checkStatus(t, w, content.Status{
+		Ref:    ref,
+		Offset: size,
+		Total:  size,
+	}, d, t1, t2, t1, t2)
+
+	if err := w.Commit(ctx2, size, d); err != nil {
+		t.Fatal(err)
+	}
+	t3 := time.Now()
+
+	info := content.Info{
+		Digest: d,
+		Size:   size,
+	}
+	if err := checkContent(ctx, cs, d, info, t1, t3, t1, t3); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := checkContent(ctx2, cs, d, info, t1, t3, t1, t3); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func checkCrossNSAppend(ctx context.Context, t *testing.T, cs content.Store) {
+	wrap, ok := ctx.Value(wrapperKey{}).(ContextWrapper)
+	if !ok {
+		t.Skip("multiple contexts not supported")
+	}
+
+	var size int64 = 1000
+	b, d := createContent(size)
+	ref := fmt.Sprintf("ref-%d", size)
+	t1 := time.Now()
+
+	if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), size, d); err != nil {
+		t.Fatal(err)
+	}
+
+	ctx2, done, err := wrap(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer done()
+
+	extra := []byte("appended bytes")
+	size2 := size + int64(len(extra))
+	b2 := make([]byte, size2)
+	copy(b2[:size], b)
+	copy(b2[size:], extra)
+	d2 := digest.FromBytes(b2)
+
+	w, err := cs.Writer(ctx2, ref, size, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t2 := time.Now()
+
+	checkStatus(t, w, content.Status{
+		Ref:    ref,
+		Offset: size,
+		Total:  size,
+	}, d, t1, t2, t1, t2)
+
+	if _, err := w.Write(extra); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := w.Commit(ctx2, size2, d2); err != nil {
+		t.Fatal(err)
+	}
+	t3 := time.Now()
+
+	info := content.Info{
+		Digest: d,
+		Size:   size,
+	}
+	if err := checkContent(ctx, cs, d, info, t1, t3, t1, t3); err != nil {
+		t.Fatal(err)
+	}
+
+	info2 := content.Info{
+		Digest: d2,
+		Size:   size2,
+	}
+	if err := checkContent(ctx2, cs, d2, info2, t1, t3, t1, t3); err != nil {
+		t.Fatal(err)
+	}
+
+}
+
 func checkStatus(t *testing.T, w content.Writer, expected content.Status, d digest.Digest, preStart, postStart, preUpdate, postUpdate time.Time) {
 	t.Helper()
 	st, err := w.Status()
@@ -564,6 +722,27 @@ func checkInfo(ctx context.Context, cs content.Store, d digest.Digest, expected
 
 	return nil
 }
+func checkContent(ctx context.Context, cs content.Store, d digest.Digest, expected content.Info, c1, c2, u1, u2 time.Time) error {
+	if err := checkInfo(ctx, cs, d, expected, c1, c2, u1, u2); err != nil {
+		return err
+	}
+
+	b, err := content.ReadBlob(ctx, cs, d)
+	if err != nil {
+		return errors.Wrap(err, "failed to read blob")
+	}
+
+	if int64(len(b)) != expected.Size {
+		return errors.Errorf("wrong blob size %d, expected %d", len(b), expected.Size)
+	}
+
+	actual := digest.FromBytes(b)
+	if actual != d {
+		return errors.Errorf("wrong digest %s, expected %s", actual, d)
+	}
+
+	return nil
+}
 
 var contentSeed int64