github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

content: change Writer/ReaderAt to take OCI

Browse Source 
This change allows implementations to resolve the location of the actual data
using OCI descriptor fields such as MediaType.

No OCI descriptor field is written to the store.

No change on gRPC API.

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>
This commit is contained in:
Akihiro Suda 2018-05-21 16:31:26 +09:00
parent e4ad710ce8
commit d88de4a34f
32 changed files with 279 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
client_test.go
View File

@ -214,7 +214,7 @@ func TestImagePullAllPlatforms(t *testing.T) {
}
// check if childless data type has blob in content store
for _, desc := range children {
ra, err := cs.ReaderAt(ctx, desc.Digest)
ra, err := cs.ReaderAt(ctx, desc)
if err != nil {
t.Fatal(err)
}
@ -275,7 +275,7 @@ func TestImagePullSomePlatforms(t *testing.T) {
// check if childless data type has blob in content store
for _, desc := range children {
ra, err := cs.ReaderAt(ctx, desc.Digest)
ra, err := cs.ReaderAt(ctx, desc)
if err != nil {
t.Fatal(err)
}

10
cmd/ctr/commands/content/content.go
View File

@ -72,7 +72,7 @@ var (
}
defer cancel()
cs := client.ContentStore()
ra, err := cs.ReaderAt(ctx, dgst)
ra, err := cs.ReaderAt(ctx, ocispec.Descriptor{Digest: dgst})
if err != nil {
return err
}
@ -121,7 +121,7 @@ var (
// TODO(stevvooe): Allow ingest to be reentrant. Currently, we expect
// all data to be written in a single invocation. Allow multiple writes
// to the same transaction key followed by a commit.
return content.WriteBlob(ctx, cs, ref, os.Stdin, expectedSize, expectedDigest)
return content.WriteBlob(ctx, cs, ref, os.Stdin, ocispec.Descriptor{Size: expectedSize, Digest: expectedDigest})
},
}
@ -314,7 +314,7 @@ var (
}
defer cancel()
cs := client.ContentStore()
ra, err := cs.ReaderAt(ctx, dgst)
ra, err := cs.ReaderAt(ctx, ocispec.Descriptor{Digest: dgst})
if err != nil {
return err
}
@ -326,7 +326,7 @@ var (
}
defer nrc.Close()
wr, err := cs.Writer(ctx, "edit-"+object, 0, "") // TODO(stevvooe): Choose a better key?
wr, err := cs.Writer(ctx, content.WithRef("edit-"+object)) // TODO(stevvooe): Choose a better key?
if err != nil {
return err
}
@ -482,7 +482,7 @@ var (
Size: info.Size,
}
ra, err := cs.ReaderAt(ctx, dgst)
ra, err := cs.ReaderAt(ctx, desc)
if err != nil {
return err
}

2
cmd/ctr/commands/snapshots/snapshots.go
View File

@ -167,7 +167,7 @@ var diffCommand = cli.Command{
}
}
ra, err := client.ContentStore().ReaderAt(ctx, desc.Digest)
ra, err := client.ContentStore().ReaderAt(ctx, desc)
if err != nil {
return err
}

13
container_opts_unix.go
View File

@ -36,9 +36,9 @@ import (
"github.com/containerd/containerd/runtime/linux/runctypes"
"github.com/gogo/protobuf/proto"
protobuf "github.com/gogo/protobuf/types"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
"github.com/opencontainers/image-spec/specs-go/v1"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -50,10 +50,9 @@ func WithCheckpoint(im Image, snapshotKey string) NewContainerOpts {
return func(ctx context.Context, client *Client, c *containers.Container) error {
var (
desc = im.Target()
id = desc.Digest
store = client.ContentStore()
)
index, err := decodeIndex(ctx, store, id)
index, err := decodeIndex(ctx, store, desc)
if err != nil {
return err
}
@ -80,7 +79,7 @@ func WithCheckpoint(im Image, snapshotKey string) NewContainerOpts {
}
c.Image = index.Annotations["image.name"]
case images.MediaTypeContainerd1CheckpointConfig:
data, err := content.ReadBlob(ctx, store, m.Digest)
data, err := content.ReadBlob(ctx, store, m)
if err != nil {
return errors.Wrap(err, "unable to read checkpoint config")
}
@ -113,7 +112,7 @@ func WithTaskCheckpoint(im Image) NewTaskOpts {
return func(ctx context.Context, c *Client, info *TaskInfo) error {
desc := im.Target()
id := desc.Digest
index, err := decodeIndex(ctx, c.ContentStore(), id)
index, err := decodeIndex(ctx, c.ContentStore(), desc)
if err != nil {
return err
}
@ -131,9 +130,9 @@ func WithTaskCheckpoint(im Image) NewTaskOpts {
}
}
func decodeIndex(ctx context.Context, store content.Provider, id digest.Digest) (*v1.Index, error) {
func decodeIndex(ctx context.Context, store content.Provider, desc ocispec.Descriptor) (*v1.Index, error) {
var index v1.Index
p, err := content.ReadBlob(ctx, store, id)
p, err := content.ReadBlob(ctx, store, desc)
if err != nil {
return nil, err
}

39
content/content.go
View File

@ -22,6 +22,7 @@ import (
"time"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
// ReaderAt extends the standard io.ReaderAt interface with reporting of Size and io.Closer
@ -33,12 +34,16 @@ type ReaderAt interface {
// Provider provides a reader interface for specific content
type Provider interface {
ReaderAt(ctx context.Context, dgst digest.Digest) (ReaderAt, error)
// ReaderAt only requires desc.Digest to be set.
// Other fields in the descriptor may be used internally for resolving
// the location of the actual data.
ReaderAt(ctx context.Context, dec ocispec.Descriptor) (ReaderAt, error)
}
// Ingester writes content
type Ingester interface {
Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (Writer, error)
// Some implementations require WithRef to be included in opts.
Writer(ctx context.Context, opts ...WriterOpt) (Writer, error)
}
// Info holds content specific information
@ -142,3 +147,33 @@ func WithLabels(labels map[string]string) Opt {
return nil
}
}
// WriterOpts is internally used by WriterOpt.
type WriterOpts struct {
Ref string
Desc ocispec.Descriptor
}
// WriterOpt is used for passing options to Ingester.Writer.
type WriterOpt func(*WriterOpts) error
// WithDescriptor specifies an OCI descriptor.
// Writer may optionally use the descriptor internally for resolving
// the location of the actual data.
// Write does not require any field of desc to be set.
// If the data size is unknown, desc.Size should be set to 0.
// Some implementations may also accept negative values as "unknown".
func WithDescriptor(desc ocispec.Descriptor) WriterOpt {
return func(opts *WriterOpts) error {
opts.Desc = desc
return nil
}
}
// WithRef specifies a ref string.
func WithRef(ref string) WriterOpt {
return func(opts *WriterOpts) error {
opts.Ref = ref
return nil
}
}

15
content/helpers.go
View File

@ -26,6 +26,7 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -45,8 +46,8 @@ func NewReader(ra ReaderAt) io.Reader {
// ReadBlob retrieves the entire contents of the blob from the provider.
//
// Avoid using this for large blobs, such as layers.
func ReadBlob(ctx context.Context, provider Provider, dgst digest.Digest) ([]byte, error) {
ra, err := provider.ReaderAt(ctx, dgst)
func ReadBlob(ctx context.Context, provider Provider, desc ocispec.Descriptor) ([]byte, error) {
ra, err := provider.ReaderAt(ctx, desc)
if err != nil {
return nil, err
}
@ -65,8 +66,8 @@ func ReadBlob(ctx context.Context, provider Provider, dgst digest.Digest) ([]byt
// This is useful when the digest and size are known beforehand.
//
// Copy is buffered, so no need to wrap reader in buffered io.
func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, size int64, expected digest.Digest, opts ...Opt) error {
cw, err := OpenWriter(ctx, cs, ref, size, expected)
func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, desc ocispec.Descriptor, opts ...Opt) error {
cw, err := OpenWriter(ctx, cs, WithRef(ref), WithDescriptor(desc))
if err != nil {
if !errdefs.IsAlreadyExists(err) {
return err
@ -76,19 +77,19 @@ func WriteBlob(ctx context.Context, cs Ingester, ref string, r io.Reader, size i
}
defer cw.Close()
return Copy(ctx, cw, r, size, expected, opts...)
return Copy(ctx, cw, r, desc.Size, desc.Digest, opts...)
}
// OpenWriter opens a new writer for the given reference, retrying if the writer
// is locked until the reference is available or returns an error.
func OpenWriter(ctx context.Context, cs Ingester, ref string, size int64, expected digest.Digest) (Writer, error) {
func OpenWriter(ctx context.Context, cs Ingester, opts ...WriterOpt) (Writer, error) {
var (
cw Writer
err error
retry = 16
)
for {
cw, err = cs.Writer(ctx, ref, size, expected)
cw, err = cs.Writer(ctx, opts...)
if err != nil {
if !errdefs.IsUnavailable(err) {
return nil, err

28
content/local/store.go
View File

@ -34,6 +34,7 @@ import (
"github.com/containerd/containerd/filters"
"github.com/containerd/containerd/log"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -119,15 +120,15 @@ func (s *store) info(dgst digest.Digest, fi os.FileInfo, labels map[string]strin
}
// ReaderAt returns an io.ReaderAt for the blob.
func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
p := s.blobPath(dgst)
func (s *store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
p := s.blobPath(desc.Digest)
fi, err := os.Stat(p)
if err != nil {
if !os.IsNotExist(err) {
return nil, err
}
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", dgst, p)
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p)
}
fp, err := os.Open(p)
@ -136,7 +137,7 @@ func (s *store) ReaderAt(ctx context.Context, dgst digest.Digest) (content.Reade
return nil, err
}
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", dgst, p)
return nil, errors.Wrapf(errdefs.ErrNotFound, "blob %s expected at %s", desc.Digest, p)
}
return sizeReaderAt{size: fi.Size(), fp: fp}, nil
@ -400,11 +401,22 @@ func (s *store) total(ingestPath string) int64 {
// ref at a time.
//
// The argument `ref` is used to uniquely identify a long-lived writer transaction.
func (s *store) Writer(ctx context.Context, ref string, total int64, expected digest.Digest) (content.Writer, error) {
func (s *store) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
// TODO(AkihiroSuda): we could create a random string or one calculated based on the context
// https://github.com/containerd/containerd/issues/2129#issuecomment-380255019
if wOpts.Ref == "" {
return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty")
}
var lockErr error
for count := uint64(0); count < 10; count++ {
time.Sleep(time.Millisecond * time.Duration(rand.Intn(1<<count)))
if err := tryLock(ref); err != nil {
if err := tryLock(wOpts.Ref); err != nil {
if !errdefs.IsUnavailable(err) {
return nil, err
}
@ -420,9 +432,9 @@ func (s *store) Writer(ctx context.Context, ref string, total int64, expected di
return nil, lockErr
}
w, err := s.writer(ctx, ref, total, expected)
w, err := s.writer(ctx, wOpts.Ref, wOpts.Desc.Size, wOpts.Desc.Digest)
if err != nil {
unlock(ref)
unlock(wOpts.Ref)
return nil, err
}

24
content/local/store_test.go
View File

@ -39,6 +39,7 @@ import (
"github.com/containerd/containerd/pkg/testutil"
"github.com/gotestyourself/gotestyourself/assert"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type memoryLabelStore struct {
@ -108,7 +109,7 @@ func TestContentWriter(t *testing.T) {
t.Fatal("ingest dir should be created", err)
}
cw, err := cs.Writer(ctx, "myref", 0, "")
cw, err := cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
t.Fatal(err)
}
@ -117,13 +118,13 @@ func TestContentWriter(t *testing.T) {
}
// reopen, so we can test things
cw, err = cs.Writer(ctx, "myref", 0, "")
cw, err = cs.Writer(ctx, content.WithRef("myref"))
if err != nil {
t.Fatal(err)
}
// make sure that second resume also fails
if _, err = cs.Writer(ctx, "myref", 0, ""); err == nil {
if _, err = cs.Writer(ctx, content.WithRef("myref")); err == nil {
// TODO(stevvooe): This also works across processes. Need to find a way
// to test that, as well.
t.Fatal("no error on second resume")
@ -166,7 +167,7 @@ func TestContentWriter(t *testing.T) {
t.Fatal(err)
}
cw, err = cs.Writer(ctx, "aref", 0, "")
cw, err = cs.Writer(ctx, content.WithRef("aref"))
if err != nil {
t.Fatal(err)
}
@ -346,7 +347,8 @@ func checkBlobPath(t *testing.T, cs content.Store, dgst digest.Digest) string {
}
func checkWrite(ctx context.Context, t checker, cs content.Store, dgst digest.Digest, p []byte) digest.Digest {
if err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p), int64(len(p)), dgst); err != nil {
if err := content.WriteBlob(ctx, cs, dgst.String(), bytes.NewReader(p),
ocispec.Descriptor{Size: int64(len(p)), Digest: dgst}); err != nil {
t.Fatal(err)
}
@ -365,25 +367,25 @@ func TestWriterTruncateRecoversFromIncompleteWrite(t *testing.T) {
defer cancel()
ref := "ref"
content := []byte("this is the content")
total := int64(len(content))
contentB := []byte("this is the content")
total := int64(len(contentB))
setupIncompleteWrite(ctx, t, cs, ref, total)
writer, err := cs.Writer(ctx, ref, total, "")
writer, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: total}))
assert.NilError(t, err)
assert.NilError(t, writer.Truncate(0))
_, err = writer.Write(content)
_, err = writer.Write(contentB)
assert.NilError(t, err)
dgst := digest.FromBytes(content)
dgst := digest.FromBytes(contentB)
err = writer.Commit(ctx, total, dgst)
assert.NilError(t, err)
}
func setupIncompleteWrite(ctx context.Context, t *testing.T, cs content.Store, ref string, total int64) {
writer, err := cs.Writer(ctx, ref, total, "")
writer, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: total}))
assert.NilError(t, err)
_, err = writer.Write([]byte("bad data"))

21
content/proxy/content_store.go
View File

@ -25,6 +25,7 @@ import (
"github.com/containerd/containerd/errdefs"
protobuftypes "github.com/gogo/protobuf/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type proxyContentStore struct {
@ -88,15 +89,16 @@ func (pcs *proxyContentStore) Delete(ctx context.Context, dgst digest.Digest) er
return nil
}
func (pcs *proxyContentStore) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
i, err := pcs.Info(ctx, dgst)
// ReaderAt ignores MediaType.
func (pcs *proxyContentStore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
i, err := pcs.Info(ctx, desc.Digest)
if err != nil {
return nil, err
}
return &remoteReaderAt{
ctx: ctx,
digest: dgst,
digest: desc.Digest,
size: i.Size,
client: pcs.client,
}, nil
@ -157,14 +159,21 @@ func (pcs *proxyContentStore) ListStatuses(ctx context.Context, filters ...strin
return statuses, nil
}
func (pcs *proxyContentStore) Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (content.Writer, error) {
wrclient, offset, err := pcs.negotiate(ctx, ref, size, expected)
// Writer ignores MediaType.
func (pcs *proxyContentStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
wrclient, offset, err := pcs.negotiate(ctx, wOpts.Ref, wOpts.Desc.Size, wOpts.Desc.Digest)
if err != nil {
return nil, errdefs.FromGRPC(err)
}
return &remoteWriter{
ref: ref,
ref: wOpts.Ref,
client: wrclient,
offset: offset,
}, nil

35
content/testsuite/testsuite.go
View File

@ -33,6 +33,7 @@ import (
"github.com/containerd/containerd/pkg/testutil"
"github.com/gotestyourself/gotestyourself/assert"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -122,28 +123,28 @@ var labels = map[string]string{
func checkContentStoreWriter(ctx context.Context, t *testing.T, cs content.Store) {
c1, d1 := createContent(256)
w1, err := cs.Writer(ctx, "c1", 0, "")
w1, err := cs.Writer(ctx, content.WithRef("c1"))
if err != nil {
t.Fatal(err)
}
defer w1.Close()
c2, d2 := createContent(256)
w2, err := cs.Writer(ctx, "c2", int64(len(c2)), "")
w2, err := cs.Writer(ctx, content.WithRef("c2"), content.WithDescriptor(ocispec.Descriptor{Size: int64(len(c2))}))
if err != nil {
t.Fatal(err)
}
defer w2.Close()
c3, d3 := createContent(256)
w3, err := cs.Writer(ctx, "c3", 0, d3)
w3, err := cs.Writer(ctx, content.WithRef("c3"), content.WithDescriptor(ocispec.Descriptor{Digest: d3}))
if err != nil {
t.Fatal(err)
}
defer w3.Close()
c4, d4 := createContent(256)
w4, err := cs.Writer(ctx, "c4", int64(len(c4)), d4)
w4, err := cs.Writer(ctx, content.WithRef("c4"), content.WithDescriptor(ocispec.Descriptor{Size: int64(len(c4)), Digest: d4}))
if err != nil {
t.Fatal(err)
}
@ -226,7 +227,7 @@ func checkResumeWriter(ctx context.Context, t *testing.T, cs content.Store) {
)
preStart := time.Now()
w1, err := cs.Writer(ctx, ref, 256, dgst)
w1, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: 256, Digest: dgst}))
if err != nil {
t.Fatal(err)
}
@ -247,7 +248,7 @@ func checkResumeWriter(ctx context.Context, t *testing.T, cs content.Store) {
checkStatus(t, w1, expected, dgstFirst, preStart, postStart, preUpdate, postUpdate)
assert.NilError(t, w1.Close(), "close first writer")
w2, err := cs.Writer(ctx, ref, 256, dgst)
w2, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: 256, Digest: dgst}))
if err != nil {
t.Fatal(err)
}
@ -284,7 +285,7 @@ func checkUpdateStatus(ctx context.Context, t *testing.T, cs content.Store) {
c1, d1 := createContent(256)
preStart := time.Now()
w1, err := cs.Writer(ctx, "c1", 256, d1)
w1, err := cs.Writer(ctx, content.WithRef("c1"), content.WithDescriptor(ocispec.Descriptor{Size: 256, Digest: d1}))
if err != nil {
t.Fatal(err)
}
@ -352,7 +353,7 @@ func checkUpdateStatus(ctx context.Context, t *testing.T, cs content.Store) {
func checkLabels(ctx context.Context, t *testing.T, cs content.Store) {
c1, d1 := createContent(256)
w1, err := cs.Writer(ctx, "c1", 256, d1)
w1, err := cs.Writer(ctx, content.WithRef("c1"), content.WithDescriptor(ocispec.Descriptor{Size: 256, Digest: d1}))
if err != nil {
t.Fatal(err)
}
@ -427,7 +428,7 @@ func checkResume(rf func(context.Context, content.Writer, []byte, int64, int64,
limit := int64(float64(size) * tp)
ref := fmt.Sprintf("ref-%d-%d", i, j)
w, err := cs.Writer(ctx, ref, size, d)
w, err := cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: size, Digest: d}))
if err != nil {
t.Fatal(err)
}
@ -441,7 +442,7 @@ func checkResume(rf func(context.Context, content.Writer, []byte, int64, int64,
t.Fatal(err)
}
w, err = cs.Writer(ctx, ref, size, d)
w, err = cs.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: size, Digest: d}))
if err != nil {
t.Fatal(err)
}
@ -530,7 +531,7 @@ func checkSmallBlob(ctx context.Context, t *testing.T, store content.Store) {
blobSize := int64(len(blob))
blobDigest := digest.FromBytes(blob)
// test write
w, err := store.Writer(ctx, t.Name(), blobSize, blobDigest)
w, err := store.Writer(ctx, content.WithRef(t.Name()), content.WithDescriptor(ocispec.Descriptor{Size: blobSize, Digest: blobDigest}))
if err != nil {
t.Fatal(err)
}
@ -545,7 +546,7 @@ func checkSmallBlob(ctx context.Context, t *testing.T, store content.Store) {
}
// test read.
readSize := blobSize + 1
ra, err := store.ReaderAt(ctx, blobDigest)
ra, err := store.ReaderAt(ctx, ocispec.Descriptor{Digest: blobDigest})
if err != nil {
t.Fatal(err)
}
@ -575,7 +576,7 @@ func checkCrossNSShare(ctx context.Context, t *testing.T, cs content.Store) {
ref := fmt.Sprintf("ref-%d", size)
t1 := time.Now()
if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), size, d); err != nil {
if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), ocispec.Descriptor{Size: size, Digest: d}); err != nil {
t.Fatal(err)
}
@ -585,7 +586,7 @@ func checkCrossNSShare(ctx context.Context, t *testing.T, cs content.Store) {
}
defer done(ctx2)
w, err := cs.Writer(ctx2, ref, size, d)
w, err := cs.Writer(ctx2, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: size, Digest: d}))
if err != nil {
t.Fatal(err)
}
@ -626,7 +627,7 @@ func checkCrossNSAppend(ctx context.Context, t *testing.T, cs content.Store) {
ref := fmt.Sprintf("ref-%d", size)
t1 := time.Now()
if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), size, d); err != nil {
if err := content.WriteBlob(ctx, cs, ref, bytes.NewReader(b), ocispec.Descriptor{Size: size, Digest: d}); err != nil {
t.Fatal(err)
}
@ -643,7 +644,7 @@ func checkCrossNSAppend(ctx context.Context, t *testing.T, cs content.Store) {
copy(b2[size:], extra)
d2 := digest.FromBytes(b2)
w, err := cs.Writer(ctx2, ref, size, d)
w, err := cs.Writer(ctx2, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{Size: size, Digest: d}))
if err != nil {
t.Fatal(err)
}
@ -768,7 +769,7 @@ func checkContent(ctx context.Context, cs content.Store, d digest.Digest, expect
return err
}
b, err := content.ReadBlob(ctx, cs, d)
b, err := content.ReadBlob(ctx, cs, ocispec.Descriptor{Digest: d})
if err != nil {
return errors.Wrap(err, "failed to read blob")
}

2
diff/apply/apply.go
View File

@ -73,7 +73,7 @@ func (s *fsApplier) Apply(ctx context.Context, desc ocispec.Descriptor, mounts [
var ocidesc ocispec.Descriptor
if err := mount.WithTempMount(ctx, mounts, func(root string) error {
ra, err := s.store.ReaderAt(ctx, desc.Digest)
ra, err := s.store.ReaderAt(ctx, desc)
if err != nil {
return errors.Wrap(err, "failed to get reader from content store")
}

6
diff/walking/differ.go
View File

@ -86,7 +86,11 @@ func (s *walkingDiff) Compare(ctx context.Context, lower, upper []mount.Mount, o
config.Reference = uniqueRef()
}
cw, err := s.store.Writer(ctx, config.Reference, 0, "")
cw, err := s.store.Writer(ctx,
content.WithRef(config.Reference),
content.WithDescriptor(ocispec.Descriptor{
MediaType: config.MediaType, // most contentstore implementations just ignore this
}))
if err != nil {
return errors.Wrap(err, "failed to open writer")
}

2
diff/windows/windows.go
View File

@ -105,7 +105,7 @@ func (s windowsDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts
return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
}
ra, err := s.store.ReaderAt(ctx, desc.Digest)
ra, err := s.store.ReaderAt(ctx, desc)
if err != nil {
return emptyDesc, errors.Wrap(err, "failed to get reader from content store")
}

16
images/image.go
View File

@ -143,7 +143,7 @@ func Manifest(ctx context.Context, provider content.Provider, image ocispec.Desc
if err := Walk(ctx, HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
switch desc.MediaType {
case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
p, err := content.ReadBlob(ctx, provider, desc)
if err != nil {
return nil, err
}
@ -159,7 +159,7 @@ func Manifest(ctx context.Context, provider content.Provider, image ocispec.Desc
}
if desc.Platform == nil {
p, err := content.ReadBlob(ctx, provider, manifest.Config.Digest)
p, err := content.ReadBlob(ctx, provider, manifest.Config)
if err != nil {
return nil, err
}
@ -180,7 +180,7 @@ func Manifest(ctx context.Context, provider content.Provider, image ocispec.Desc
return nil, nil
case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
p, err := content.ReadBlob(ctx, provider, desc)
if err != nil {
return nil, err
}
@ -240,7 +240,7 @@ func Platforms(ctx context.Context, provider content.Provider, image ocispec.Des
switch desc.MediaType {
case MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
p, err := content.ReadBlob(ctx, provider, desc)
if err != nil {
return nil, err
}
@ -283,7 +283,7 @@ func Check(ctx context.Context, provider content.Provider, image ocispec.Descrip
required = append([]ocispec.Descriptor{mfst.Config}, mfst.Layers...)
for _, desc := range required {
ra, err := provider.ReaderAt(ctx, desc.Digest)
ra, err := provider.ReaderAt(ctx, desc)
if err != nil {
if errdefs.IsNotFound(err) {
missing = append(missing, desc)
@ -305,7 +305,7 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr
var descs []ocispec.Descriptor
switch desc.MediaType {
case MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
p, err := content.ReadBlob(ctx, provider, desc)
if err != nil {
return nil, err
}
@ -320,7 +320,7 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr
descs = append(descs, manifest.Config)
descs = append(descs, manifest.Layers...)
case MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
p, err := content.ReadBlob(ctx, provider, desc.Digest)
p, err := content.ReadBlob(ctx, provider, desc)
if err != nil {
return nil, err
}
@ -351,7 +351,7 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr
// These are used to verify that a set of layers unpacked to the expected
// values.
func RootFS(ctx context.Context, provider content.Provider, configDesc ocispec.Descriptor) ([]digest.Digest, error) {
p, err := content.ReadBlob(ctx, provider, configDesc.Digest)
p, err := content.ReadBlob(ctx, provider, configDesc)
if err != nil {
return nil, err
}

2
images/oci/exporter.go
View File

@ -93,7 +93,7 @@ func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord {
Typeflag: tar.TypeReg,
},
CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
r, err := cs.ReaderAt(ctx, desc.Digest)
r, err := cs.ReaderAt(ctx, desc)
if err != nil {
return 0, err
}

4
images/oci/importer.go
View File

@ -140,7 +140,7 @@ func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, name
return errors.Errorf("unsupported algorithm: %s", algo)
}
dgst := digest.NewDigestFromHex(algo.String(), split[2])
return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, ocispec.Descriptor{Size: size, Digest: dgst})
}
// GetChildrenDescriptors returns children blob descriptors for the following supported types:
@ -175,7 +175,7 @@ func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispe
}
return err
}
ra, err := store.ReaderAt(ctx, desc.Digest)
ra, err := store.ReaderAt(ctx, desc)
if err != nil {
return err
}

94
metadata/content.go
View File

@ -32,6 +32,7 @@ import (
"github.com/containerd/containerd/metadata/boltutil"
"github.com/containerd/containerd/namespaces"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -337,7 +338,18 @@ func (cs *contentStore) Abort(ctx context.Context, ref string) error {
}
func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (content.Writer, error) {
func (cs *contentStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
// TODO(AkihiroSuda): we could create a random string or one calculated based on the context
// https://github.com/containerd/containerd/issues/2129#issuecomment-380255019
if wOpts.Ref == "" {
return nil, errors.Wrap(errdefs.ErrInvalidArgument, "ref must not be empty")
}
ns, err := namespaces.NamespaceRequired(ctx)
if err != nil {
return nil, err
@ -353,12 +365,12 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
)
if err := update(ctx, cs.db, func(tx *bolt.Tx) error {
var shared bool
if expected != "" {
cbkt := getBlobBucket(tx, ns, expected)
if wOpts.Desc.Digest != "" {
cbkt := getBlobBucket(tx, ns, wOpts.Desc.Digest)
if cbkt != nil {
// Add content to lease to prevent other reference removals
// from effecting this object during a provided lease
if err := addContentLease(ctx, tx, expected); err != nil {
if err := addContentLease(ctx, tx, wOpts.Desc.Digest); err != nil {
return errors.Wrap(err, "unable to lease content")
}
// Return error outside of transaction to ensure
@ -367,18 +379,18 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
return nil
}
if st, err := cs.Store.Info(ctx, expected); err == nil {
if st, err := cs.Store.Info(ctx, wOpts.Desc.Digest); err == nil {
// Ensure the expected size is the same, it is likely
// an error if the size is mismatched but the caller
// must resolve this on commit
if size == 0 || size == st.Size {
if wOpts.Desc.Size == 0 || wOpts.Desc.Size == st.Size {
shared = true
size = st.Size
wOpts.Desc.Size = st.Size
}
}
}
bkt, err := createIngestBucket(tx, ns, ref)
bkt, err := createIngestBucket(tx, ns, wOpts.Ref)
if err != nil {
return err
}
@ -390,7 +402,7 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
return err
}
bref = createKey(sid, ns, ref)
bref = createKey(sid, ns, wOpts.Ref)
if err := bkt.Put(bucketKeyRef, []byte(bref)); err != nil {
return err
}
@ -399,7 +411,7 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
}
if shared {
if err := bkt.Put(bucketKeyExpected, []byte(expected)); err != nil {
if err := bkt.Put(bucketKeyExpected, []byte(wOpts.Desc.Digest)); err != nil {
return err
}
} else {
@ -407,19 +419,21 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
// already checked against the user metadata. The content must
// be committed in the namespace before it will be seen as
// available in the current namespace.
w, err = cs.Store.Writer(ctx, bref, size, "")
desc := wOpts.Desc
desc.Digest = ""
w, err = cs.Store.Writer(ctx, content.WithRef(bref), content.WithDescriptor(desc))
}
return err
}); err != nil {
return nil, err
}
if exists {
return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", expected)
return nil, errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", wOpts.Desc.Digest)
}
return &namespacedWriter{
ctx: ctx,
ref: ref,
ref: wOpts.Ref,
namespace: ns,
db: cs.db,
provider: cs.Store,
@ -427,8 +441,7 @@ func (cs *contentStore) Writer(ctx context.Context, ref string, size int64, expe
w: w,
bref: bref,
started: time.Now(),
expected: expected,
size: size,
desc: wOpts.Desc,
}, nil
}
@ -445,10 +458,9 @@ type namespacedWriter struct {
w content.Writer
bref string
started time.Time
expected digest.Digest
size int64
bref string
started time.Time
desc ocispec.Descriptor
}
func (nw *namespacedWriter) Close() error {
@ -465,7 +477,7 @@ func (nw *namespacedWriter) Write(p []byte) (int, error) {
return 0, nil
}
if err := nw.createAndCopy(nw.ctx, nw.size); err != nil {
if err := nw.createAndCopy(nw.ctx, nw.desc); err != nil {
return 0, err
}
}
@ -477,31 +489,35 @@ func (nw *namespacedWriter) Digest() digest.Digest {
if nw.w != nil {
return nw.w.Digest()
}
return nw.expected
return nw.desc.Digest
}
func (nw *namespacedWriter) Truncate(size int64) error {
if nw.w != nil {
return nw.w.Truncate(size)
}
return nw.createAndCopy(nw.ctx, size)
desc := nw.desc
desc.Size = size
desc.Digest = ""
return nw.createAndCopy(nw.ctx, desc)
}
func (nw *namespacedWriter) createAndCopy(ctx context.Context, size int64) error {
w, err := nw.provider.Writer(ctx, nw.bref, nw.size, "")
func (nw *namespacedWriter) createAndCopy(ctx context.Context, desc ocispec.Descriptor) error {
nwDescWithoutDigest := desc
nwDescWithoutDigest.Digest = ""
w, err := nw.provider.Writer(ctx, content.WithRef(nw.bref), content.WithDescriptor(nwDescWithoutDigest))
if err != nil {
return err
}
if size > 0 {
ra, err := nw.provider.ReaderAt(ctx, nw.expected)
if desc.Size > 0 {
ra, err := nw.provider.ReaderAt(ctx, nw.desc)
if err != nil {
return err
}
defer ra.Close()
if err := content.CopyReaderAt(w, ra, size); err != nil {
if err := content.CopyReaderAt(w, ra, desc.Size); err != nil {
nw.w.Close()
nw.w = nil
return err
@ -544,14 +560,14 @@ func (nw *namespacedWriter) commit(ctx context.Context, tx *bolt.Tx, size int64,
var actual digest.Digest
if nw.w == nil {
if size != 0 && size != nw.size {
return "", errors.Errorf("%q failed size validation: %v != %v", nw.ref, nw.size, size)
if size != 0 && size != nw.desc.Size {
return "", errors.Errorf("%q failed size validation: %v != %v", nw.ref, nw.desc.Size, size)
}
if expected != "" && expected != nw.expected {
if expected != "" && expected != nw.desc.Digest {
return "", errors.Errorf("%q unexpected digest", nw.ref)
}
size = nw.size
actual = nw.expected
size = nw.desc.Size
actual = nw.desc.Digest
if getBlobBucket(tx, nw.namespace, actual) != nil {
return "", errors.Wrapf(errdefs.ErrAlreadyExists, "content %v", actual)
}
@ -601,11 +617,11 @@ func (nw *namespacedWriter) Status() (st content.Status, err error) {
if nw.w != nil {
st, err = nw.w.Status()
} else {
st.Offset = nw.size
st.Total = nw.size
st.Offset = nw.desc.Size
st.Total = nw.desc.Size
st.StartedAt = nw.started
st.UpdatedAt = nw.started
st.Expected = nw.expected
st.Expected = nw.desc.Digest
}
if err == nil {
st.Ref = nw.ref
@ -613,11 +629,11 @@ func (nw *namespacedWriter) Status() (st content.Status, err error) {
return
}
func (cs *contentStore) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
if err := cs.checkAccess(ctx, dgst); err != nil {
func (cs *contentStore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
if err := cs.checkAccess(ctx, desc.Digest); err != nil {
return nil, err
}
return cs.Store.ReaderAt(ctx, dgst)
return cs.Store.ReaderAt(ctx, desc)
}
func (cs *contentStore) checkAccess(ctx context.Context, dgst digest.Digest) error {

8
metadata/content_test.go
View File

@ -32,6 +32,7 @@ import (
"github.com/containerd/containerd/leases"
"github.com/containerd/containerd/namespaces"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)
@ -81,7 +82,8 @@ func TestContentLeased(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := content.WriteBlob(lctx, cs, "test-1", bytes.NewReader(blob), int64(len(blob)), expected); err != nil {
if err := content.WriteBlob(lctx, cs, "test-1", bytes.NewReader(blob),
ocispec.Descriptor{Size: int64(len(blob)), Digest: expected}); err != nil {
t.Fatal(err)
}
if err := checkContentLeased(lctx, db, expected); err != nil {
@ -93,7 +95,9 @@ func TestContentLeased(t *testing.T) {
t.Fatal(err)
}
if _, err := cs.Writer(lctx, "test-2", int64(len(blob)), expected); err == nil {
if _, err := cs.Writer(lctx,
content.WithRef("test-2"),
content.WithDescriptor(ocispec.Descriptor{Size: int64(len(blob)), Digest: expected})); err == nil {
t.Fatal("expected already exist error")
} else if !errdefs.IsAlreadyExists(err) {
t.Fatal(err)

4
metadata/db_test.go
View File

@ -506,7 +506,9 @@ func create(obj object, tx *bolt.Tx, is images.Store, cs content.Store, sn snaps
case testContent:
ctx := WithTransactionContext(ctx, tx)
expected := digest.FromBytes(v.data)
w, err := cs.Writer(ctx, "test-ref", int64(len(v.data)), expected)
w, err := cs.Writer(ctx,
content.WithRef("test-ref"),
content.WithDescriptor(ocispec.Descriptor{Size: int64(len(v.data)), Digest: expected}))
if err != nil {
return nil, errors.Wrap(err, "failed to create writer")
}

2
oci/spec_opts_unix.go
View File

@ -117,7 +117,7 @@ func WithImageConfig(image Image) SpecOpts {
)
switch ic.MediaType {
case v1.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config:
p, err := content.ReadBlob(ctx, image.ContentStore(), ic.Digest)
p, err := content.ReadBlob(ctx, image.ContentStore(), ic)
if err != nil {
return err
}

2
oci/spec_opts_windows.go
View File

@ -44,7 +44,7 @@ func WithImageConfig(image Image) SpecOpts {
)
switch ic.MediaType {
case v1.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config:
p, err := content.ReadBlob(ctx, image.ContentStore(), ic.Digest)
p, err := content.ReadBlob(ctx, image.ContentStore(), ic)
if err != nil {
return err
}

8
remotes/docker/schema1/converter.go
View File

@ -211,12 +211,12 @@ func (c *Converter) Convert(ctx context.Context, opts ...ConvertOpt) (ocispec.De
}
ref := remotes.MakeRefKey(ctx, desc)
if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(mb), desc.Size, desc.Digest, content.WithLabels(labels)); err != nil {
if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(mb), desc, content.WithLabels(labels)); err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config")
}
ref = remotes.MakeRefKey(ctx, config)
if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(b), config.Size, config.Digest); err != nil {
if err := content.WriteBlob(ctx, c.contentStore, ref, bytes.NewReader(b), config); err != nil {
return ocispec.Descriptor{}, errors.Wrap(err, "failed to write config")
}
@ -265,7 +265,7 @@ func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) erro
size = 0
}
cw, err := content.OpenWriter(ctx, c.contentStore, ref, size, desc.Digest)
cw, err := content.OpenWriter(ctx, c.contentStore, content.WithRef(ref), content.WithDescriptor(desc))
if err != nil {
if !errdefs.IsAlreadyExists(err) {
return err
@ -274,7 +274,7 @@ func (c *Converter) fetchBlob(ctx context.Context, desc ocispec.Descriptor) erro
// TODO: Check if blob -> diff id mapping already exists
// TODO: Check if blob empty label exists
ra, err := c.contentStore.ReaderAt(ctx, desc.Digest)
ra, err := c.contentStore.ReaderAt(ctx, desc)
if err != nil {
return err
}

4
remotes/handlers.go
View File

@ -81,7 +81,7 @@ func FetchHandler(ingester content.Ingester, fetcher Fetcher) images.HandlerFunc
func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc ocispec.Descriptor) error {
log.G(ctx).Debug("fetch")
cw, err := content.OpenWriter(ctx, ingester, MakeRefKey(ctx, desc), desc.Size, desc.Digest)
cw, err := content.OpenWriter(ctx, ingester, content.WithRef(MakeRefKey(ctx, desc)), content.WithDescriptor(desc))
if err != nil {
if errdefs.IsAlreadyExists(err) {
return nil
@ -141,7 +141,7 @@ func push(ctx context.Context, provider content.Provider, pusher Pusher, desc oc
}
defer cw.Close()
ra, err := provider.ReaderAt(ctx, desc.Digest)
ra, err := provider.ReaderAt(ctx, desc)
if err != nil {
return err
}

7
services/content/service.go
View File

@ -29,6 +29,7 @@ import (
"github.com/containerd/containerd/services"
ptypes "github.com/gogo/protobuf/types"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"google.golang.org/grpc"
@ -179,7 +180,7 @@ func (s *service) Read(req *api.ReadContentRequest, session api.Content_ReadServ
return errdefs.ToGRPC(err)
}
ra, err := s.store.ReaderAt(session.Context(), req.Digest)
ra, err := s.store.ReaderAt(session.Context(), ocispec.Descriptor{Digest: req.Digest})
if err != nil {
return errdefs.ToGRPC(err)
}
@ -334,7 +335,9 @@ func (s *service) Write(session api.Content_WriteServer) (err error) {
log.G(ctx).Debug("(*service).Write started")
// this action locks the writer for the session.
wr, err := s.store.Writer(ctx, ref, total, expected)
wr, err := s.store.Writer(ctx,
content.WithRef(ref),
content.WithDescriptor(ocispec.Descriptor{Size: total, Digest: expected}))
if err != nil {
return errdefs.ToGRPC(err)
}

9
services/tasks/local.go
View File

@ -45,6 +45,7 @@ import (
"github.com/containerd/containerd/services"
"github.com/containerd/typeurl"
ptypes "github.com/gogo/protobuf/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
@ -121,7 +122,11 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
if r.Checkpoint.MediaType != images.MediaTypeContainerd1Checkpoint {
return nil, fmt.Errorf("unsupported checkpoint type %q", r.Checkpoint.MediaType)
}
reader, err := l.store.ReaderAt(ctx, r.Checkpoint.Digest)
reader, err := l.store.ReaderAt(ctx, ocispec.Descriptor{
MediaType: r.Checkpoint.MediaType,
Digest: r.Checkpoint.Digest,
Size: r.Checkpoint.Size_,
})
if err != nil {
return nil, err
}
@ -573,7 +578,7 @@ func getTasksMetrics(ctx context.Context, filter filters.Filter, tasks []runtime
}
func (l *local) writeContent(ctx context.Context, mediaType, ref string, r io.Reader) (*types.Descriptor, error) {
writer, err := l.store.Writer(ctx, ref, 0, "")
writer, err := l.store.Writer(ctx, content.WithRef(ref), content.WithDescriptor(ocispec.Descriptor{MediaType: mediaType}))
if err != nil {
return nil, err
}

2
task.go
View File

@ -597,7 +597,7 @@ func (t *task) writeIndex(ctx context.Context, index *v1.Index) (d v1.Descriptor
}
func writeContent(ctx context.Context, store content.Ingester, mediaType, ref string, r io.Reader, opts ...content.Opt) (d v1.Descriptor, err error) {
writer, err := store.Writer(ctx, ref, 0, "")
writer, err := store.Writer(ctx, content.WithRef(ref))
if err != nil {
return d, err
}

4
vendor.conf
View File

@ -43,8 +43,8 @@ github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
github.com/gotestyourself/gotestyourself 44dbf532bbf5767611f6f2a61bded572e337010a
github.com/google/go-cmp v0.1.0
# cri dependencies
github.com/containerd/cri b68fb075d49aa1c2885f45f2467142666c244f4a
# #2135: cri is temporarily forked because of circular dependency. will be fixed immediately in a follow-up PR.
github.com/containerd/cri 6e975823be192ad19f2ce7afcf6c57b88a991c30 https://github.com/AkihiroSuda/cri-containerd.git
github.com/containerd/go-cni f2d7272f12d045b16ed924f50e91f9f9cecc55a7
github.com/blang/semver v3.1.0
github.com/containernetworking/cni v0.6.0

17
vendor/github.com/containerd/cri/pkg/containerd/importer/importer.go generated vendored
View File

@ -254,23 +254,24 @@ func writeDockerSchema2Manifest(ctx context.Context, cs content.Ingester, manife
for i, ch := range manifest.Layers {
labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = ch.Digest.String()
}
if err := content.WriteBlob(ctx, cs, "manifest-"+manifestDigest.String(), manifestBytesR,
int64(len(manifestBytes)), manifestDigest, content.WithLabels(labels)); err != nil {
return nil, err
}
desc := &ocispec.Descriptor{
desc := ocispec.Descriptor{
MediaType: images.MediaTypeDockerSchema2Manifest,
Digest: manifestDigest,
Size: int64(len(manifestBytes)),
}
if err := content.WriteBlob(ctx, cs, "manifest-"+manifestDigest.String(), manifestBytesR,
desc, content.WithLabels(labels)); err != nil {
return nil, err
}
if arch != "" || os != "" {
desc.Platform = &ocispec.Platform{
Architecture: arch,
OS: os,
}
}
return desc, nil
return &desc, nil
}
func onUntarManifestJSON(r io.Reader) ([]manifestDotJSON, error) {
@ -290,7 +291,7 @@ func onUntarLayerTar(ctx context.Context, r io.Reader, cs content.Ingester, name
// name is like "foobar/layer.tar" ( guaranteed by isLayerTar() )
split := strings.Split(name, "/")
// note: split[0] is not expected digest here
cw, err := cs.Writer(ctx, "layer-"+split[0], size, "")
cw, err := cs.Writer(ctx, content.WithRef("layer-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size}))
if err != nil {
return nil, err
}
@ -311,7 +312,7 @@ func onUntarDotJSON(ctx context.Context, r io.Reader, cs content.Ingester, name
config.desc.Size = size
// name is like "foobar.json" ( guaranteed by is DotJSON() )
split := strings.Split(name, ".")
cw, err := cs.Writer(ctx, "config-"+split[0], size, "")
cw, err := cs.Writer(ctx, content.WithRef("config-"+split[0]), content.WithDescriptor(ocispec.Descriptor{Size: size}))
if err != nil {
return nil, err
}

20
vendor/github.com/containerd/cri/pkg/server/events.go generated vendored
View File

@ -116,15 +116,16 @@ func convertEvent(e *gogotypes.Any) (string, interface{}, error) {
}
// start starts the event monitor which monitors and handles all container events. It returns
// a channel for the caller to wait for the event monitor to stop. start must be called after
// subscribe.
func (em *eventMonitor) start() (<-chan struct{}, error) {
// an error channel for the caller to wait for stop errors from the event monitor.
// start must be called after subscribe.
func (em *eventMonitor) start() <-chan error {
errCh := make(chan error)
if em.ch == nil || em.errCh == nil {
return nil, errors.New("event channel is nil")
panic("event channel is nil")
}
closeCh := make(chan struct{})
backOffCheckCh := em.backOff.start()
go func() {
defer close(errCh)
for {
select {
case e := <-em.ch:
@ -144,8 +145,11 @@ func (em *eventMonitor) start() (<-chan struct{}, error) {
em.backOff.enBackOff(cID, evt)
}
case err := <-em.errCh:
logrus.WithError(err).Error("Failed to handle event stream")
close(closeCh)
// Close errCh in defer directly if there is no error.
if err != nil {
logrus.WithError(err).Errorf("Failed to handle event stream")
errCh <- err
}
return
case <-backOffCheckCh:
cIDs := em.backOff.getExpiredContainers()
@ -162,7 +166,7 @@ func (em *eventMonitor) start() (<-chan struct{}, error) {
}
}
}()
return closeCh, nil
return errCh
}
// stop stops the event monitor. It will close the event channel.

2
vendor/github.com/containerd/cri/pkg/server/helpers.go generated vendored
View File

@ -345,7 +345,7 @@ func getImageInfo(ctx context.Context, image containerd.Image) (*imageInfo, erro
}
id := desc.Digest.String()
rb, err := content.ReadBlob(ctx, image.ContentStore(), desc.Digest)
rb, err := content.ReadBlob(ctx, image.ContentStore(), desc)
if err != nil {
return nil, errors.Wrap(err, "failed to read image config from content store")
}

40
vendor/github.com/containerd/cri/pkg/server/service.go generated vendored
View File

@ -19,6 +19,7 @@ package server
import (
"fmt"
"io"
"net/http"
"path/filepath"
"time"
@ -179,10 +180,7 @@ func (c *criService) Run() error {
// Start event handler.
logrus.Info("Start event monitor")
eventMonitorCloseCh, err := c.eventMonitor.start()
if err != nil {
return errors.Wrap(err, "failed to start event monitor")
}
eventMonitorErrCh := c.eventMonitor.start()
// Start snapshot stats syncer, it doesn't need to be stopped.
logrus.Info("Start snapshots syncer")
@ -195,27 +193,32 @@ func (c *criService) Run() error {
// Start streaming server.
logrus.Info("Start streaming server")
streamServerCloseCh := make(chan struct{})
streamServerErrCh := make(chan error)
go func() {
if err := c.streamServer.Start(true); err != nil {
defer close(streamServerErrCh)
if err := c.streamServer.Start(true); err != nil && err != http.ErrServerClosed {
logrus.WithError(err).Error("Failed to start streaming server")
streamServerErrCh <- err
}
close(streamServerCloseCh)
}()
// Set the server as initialized. GRPC services could start serving traffic.
c.initialized.Set()
var eventMonitorErr, streamServerErr error
// Stop the whole CRI service if any of the critical service exits.
select {
case <-eventMonitorCloseCh:
case <-streamServerCloseCh:
case eventMonitorErr = <-eventMonitorErrCh:
case streamServerErr = <-streamServerErrCh:
}
if err := c.Close(); err != nil {
return errors.Wrap(err, "failed to stop cri service")
}
<-eventMonitorCloseCh
// If the error is set above, err from channel must be nil here, because
// the channel is supposed to be closed. Or else, we wait and set it.
if err := <-eventMonitorErrCh; err != nil {
eventMonitorErr = err
}
logrus.Info("Event monitor stopped")
// There is a race condition with http.Server.Serve.
// When `Close` is called at the same time with `Serve`, `Close`
@ -227,18 +230,27 @@ func (c *criService) Run() error {
// is fixed.
const streamServerStopTimeout = 2 * time.Second
select {
case <-streamServerCloseCh:
case err := <-streamServerErrCh:
if err != nil {
streamServerErr = err
}
logrus.Info("Stream server stopped")
case <-time.After(streamServerStopTimeout):
logrus.Errorf("Stream server is not stopped in %q", streamServerStopTimeout)
}
if eventMonitorErr != nil {
return errors.Wrap(eventMonitorErr, "event monitor error")
}
if streamServerErr != nil {
return errors.Wrap(streamServerErr, "stream server error")
}
return nil
}
// Stop stops the CRI service.
// Close stops the CRI service.
// TODO(random-liu): Make close synchronous.
func (c *criService) Close() error {
logrus.Info("Stop CRI service")
// TODO(random-liu): Make event monitor stop synchronous.
c.eventMonitor.stop()
if err := c.streamServer.Stop(); err != nil {
return errors.Wrap(err, "failed to stop stream server")

2
vendor/github.com/containerd/cri/vendor.conf generated vendored
View File

@ -4,7 +4,7 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
github.com/containerd/cgroups fe281dd265766145e943a034aa41086474ea6130
github.com/containerd/console cb7008ab3d8359b78c5f464cb7cf160107ad5925
github.com/containerd/containerd 1e8b09cfc6825f7e6349884b5f76e86c1f04a5d4
github.com/containerd/containerd 1f8c612a6c7ef2fc8328c953fb660adce2bf0a80 https://github.com/AkihiroSuda/containerd.git
github.com/containerd/continuity 2d3749b4da569ac97ca63dccba5eee4f5ee2beab
github.com/containerd/fifo 3d5202aec260678c48179c56f40e6f38a095738c
github.com/containerd/go-cni f2d7272f12d045b16ed924f50e91f9f9cecc55a7