github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

content: change Writer/ReaderAt to take OCI

Browse Source 
This change allows implementations to resolve the location of the actual data
using OCI descriptor fields such as MediaType.

No OCI descriptor field is written to the store.

No change on gRPC API.

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
This commit is contained in:
Akihiro Suda
2018-05-21 16:31:26 +09:00
parent e4ad710ce8
commit d88de4a34f
32 changed files with 279 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
content/local/store.go
View File

@@ -34,6 +34,7 @@ import (
"github.com/containerd/containerd/filters"
"github.com/containerd/containerd/log"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@@ -119,15 +120,15 @@ func (s *store) info(dgst digest.Digest, fi os.FileInfo, labels map[string]strin
}
// ReaderAt returns an io.ReaderAt for the blob.
func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
p := s.blobPath(dgst)
func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
p := s.blobPath(desc.Digest)
fi, err := os.Stat(p)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", dgst, p)
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p)
}
fp, err := os.Open(p)
@@ -136,7 +137,7 @@ func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (content.Reade
return nil, err
}
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", dgst, p)
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p)
}
return sizeReaderAt{size: fi.Size(), fp: fp}, nil
@@ -400,11 +401,22 @@ func (s *store) total(ingestPath string) int64 {
// ref at a time.
//
// The argument `ref` is used to uniquely identify a long-lived writer transaction.
func (s *store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (content.Writer, error) {
func (s *store) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
// TODO(AkihiroSuda): we could create a random string or one calculated based on the context
// https://github.com/containerd/containerd/issues/2129#issuecomment-380255019
if wOpts.Ref == "" {
return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty")
}
var lockErr error
for count := uint64(0); count < 10; count++ {
time.Sleep(time.Millisecond * time.Duration(rand.Intn(1<<count)))
if err := tryLock(ref); err != nil {
if err := tryLock(wOpts.Ref); err != nil {
if !errdefs.IsUnavailable(err) {
return nil, err
}
@@ -420,9 +432,9 @@ func (s *store) Writer(ctx context.Context, ref string, total int64, expected di
return nil, lockErr
}
w, err := s.writer(ctx, ref, total, expected)
w, err := s.writer(ctx, wOpts.Ref, wOpts.Desc.Size, wOpts.Desc.Digest)
if err != nil {
unlock(ref)
unlock(wOpts.Ref)
return nil, err
}

24
content/local/store_test.go
View File

@@ -39,6 +39,7 @@ import (
"github.com/containerd/containerd/pkg/testutil"
"github.com/gotestyourself/gotestyourself/assert"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type memoryLabelStore struct {
@@ -108,7 +109,7 @@ func TestContentWriter(t *testing.T) {
t.Fatal("ingest dir should be created", err)
}
cw, err := cs.Writer(ctx, "myref", 0, "")
cw, err := cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
t.Fatal(err)
}
@@ -117,13 +118,13 @@ func TestContentWriter(t *testing.T) {
}
// reopen, so we can test things
cw, err = cs.Writer(ctx, "myref", 0, "")
cw, err = cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
t.Fatal(err)
}
// make sure that second resume also fails
if _, err = cs.Writer(ctx, "myref", 0, ""); err == nil {
if _, err = cs.Writer(ctx, content.WithRef("myref")); err == nil {
// TODO(stevvooe): This also works across processes. Need to find a way
// to test that, as well.
t.Fatal("no error on second resume")
@@ -166,7 +167,7 @@ func TestContentWriter(t *testing.T) {
t.Fatal(err)
}
cw, err = cs.Writer(ctx, "aref", 0, "")
cw, err = cs.Writer(ctx, content.WithRef("aref"))
if err != nil {
t.Fatal(err)
}
@@ -346,7 +347,8 @@ func checkBlobPath(t *testing.T, cs content.Store, dgst digest.Digest) string {
}
func checkWrite(ctx context.Context, t checker, cs content.Store, dgst digest.Digest, p []byte) digest.Digest {
if err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p), int64(len(p)), dgst); err != nil {
if err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p),
ocispec.Descriptor{Size: int64(len(p)), Digest: dgst}); err != nil {
t.Fatal(err)
}
@@ -365,25 +367,25 @@ func TestWriterTruncateRecoversFromIncompleteWrite(t *testing.T) {
defer cancel()
ref := "ref"
content := []byte("this is the content")
total := int64(len(content))
contentB := []byte("this is the content")
total := int64(len(contentB))
setupIncompleteWrite(ctx, t, cs, ref, total)
writer, err := cs.Writer(ctx, ref, total, "")
writer, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: total}))
assert.NilError(t, err)
assert.NilError(t, writer.Truncate(0))
_, err = writer.Write(content)
_, err = writer.Write(contentB)
assert.NilError(t, err)
dgst := digest.FromBytes(content)
dgst := digest.FromBytes(contentB)
err = writer.Commit(ctx, total, dgst)
assert.NilError(t, err)
}
func setupIncompleteWrite(ctx context.Context, t *testing.T, cs content.Store, ref string, total int64) {
writer, err := cs.Writer(ctx, ref, total, "")
writer, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: total}))
assert.NilError(t, err)
_, err = writer.Write([]byte("bad data"))