From c4387a159eb62a5c94b864b73f095b2950fd81e4 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Aug 2017 13:30:21 -0700 Subject: [PATCH 1/3] Add content test suite run to client Fix bugs in content deletion and upload status Signed-off-by: Derek McGowan --- content/local/store_test.go | 6 +++-- content/testsuite/testsuite.go | 12 ++++++---- content_test.go | 42 ++++++++++++++++++++++++++++++++++ metadata/content.go | 2 +- metadata/content_test.go | 32 ++++++++++++++------------ services/content/writer.go | 1 + 6 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 content_test.go diff --git a/content/local/store_test.go b/content/local/store_test.go index 47bb77a1f..4d7afe070 100644 --- a/content/local/store_test.go +++ b/content/local/store_test.go @@ -24,12 +24,14 @@ import ( ) func TestContent(t *testing.T) { - testsuite.ContentSuite(t, "fs", func(ctx context.Context, root string) (content.Store, func(), error) { + testsuite.ContentSuite(t, "fs", func(ctx context.Context, root string) (content.Store, func() error, error) { cs, err := NewStore(root) if err != nil { return nil, nil, err } - return cs, func() {}, nil + return cs, func() error { + return nil + }, nil }) } diff --git a/content/testsuite/testsuite.go b/content/testsuite/testsuite.go index c9f7de52d..afeb4d918 100644 --- a/content/testsuite/testsuite.go +++ b/content/testsuite/testsuite.go @@ -17,13 +17,13 @@ import ( "github.com/pkg/errors" ) -// ContentSuite runs a test suite on the snapshotter given a factory function. -func ContentSuite(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func(), error)) { +// ContentSuite runs a test suite on the content store given a factory function. +func ContentSuite(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func() error, error)) { t.Run("Writer", makeTest(t, name, storeFn, checkContentStoreWriter)) t.Run("UploadStatus", makeTest(t, name, storeFn, checkUploadStatus)) } -func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func(), error), fn func(ctx context.Context, t *testing.T, cs content.Store)) func(t *testing.T) { +func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func() error, error), fn func(ctx context.Context, t *testing.T, cs content.Store)) func(t *testing.T) { return func(t *testing.T) { ctx := namespaces.WithNamespace(context.Background(), name) @@ -37,7 +37,11 @@ func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root if err != nil { t.Fatal(err) } - defer cleanup() + defer func() { + if err := cleanup(); err != nil && !t.Failed() { + t.Fatalf("Cleanup failed: %+v", err) + } + }() defer testutil.DumpDir(t, tmpDir) fn(ctx, t, cs) diff --git a/content_test.go b/content_test.go new file mode 100644 index 000000000..6fc7818ae --- /dev/null +++ b/content_test.go @@ -0,0 +1,42 @@ +package containerd + +import ( + "context" + "testing" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/content/testsuite" + "github.com/pkg/errors" +) + +func newContentStore(ctx context.Context, root string) (content.Store, func() error, error) { + client, err := New(address) + if err != nil { + return nil, nil, err + } + + cs := client.ContentStore() + + return cs, func() error { + statuses, err := cs.ListStatuses(ctx) + if err != nil { + return err + } + for _, st := range statuses { + if err := cs.Abort(ctx, st.Ref); err != nil { + return errors.Wrapf(err, "failed to abort %s", st.Ref) + } + } + return cs.Walk(ctx, func(info content.Info) error { + return cs.Delete(ctx, info.Digest) + }) + + }, nil +} + +func TestContentClient(t *testing.T) { + if testing.Short() { + t.Skip() + } + testsuite.ContentSuite(t, "ContentClient", newContentStore) +} diff --git a/metadata/content.go b/metadata/content.go index 8ad847f83..c0b3fc795 100644 --- a/metadata/content.go +++ b/metadata/content.go @@ -170,7 +170,7 @@ func (cs *contentStore) Delete(ctx context.Context, dgst digest.Digest) error { // Just remove local reference, garbage collector is responsible for // cleaning up on disk content - return getBlobsBucket(tx, ns).Delete([]byte(dgst.String())) + return getBlobsBucket(tx, ns).DeleteBucket([]byte(dgst.String())) }) } diff --git a/metadata/content_test.go b/metadata/content_test.go index 77ef293ce..093736833 100644 --- a/metadata/content_test.go +++ b/metadata/content_test.go @@ -11,21 +11,23 @@ import ( "github.com/containerd/containerd/content/testsuite" ) -func TestContent(t *testing.T) { - testsuite.ContentSuite(t, "metadata", func(ctx context.Context, root string) (content.Store, func(), error) { - // TODO: Use mocked or in-memory store - cs, err := local.NewStore(root) - if err != nil { - return nil, nil, err - } +func createContentStore(ctx context.Context, root string) (content.Store, func() error, error) { + // TODO: Use mocked or in-memory store + cs, err := local.NewStore(root) + if err != nil { + return nil, nil, err + } - db, err := bolt.Open(filepath.Join(root, "metadata.db"), 0660, nil) - if err != nil { - return nil, nil, err - } + db, err := bolt.Open(filepath.Join(root, "metadata.db"), 0660, nil) + if err != nil { + return nil, nil, err + } - return NewContentStore(db, cs), func() { - db.Close() - }, nil - }) + return NewContentStore(db, cs), func() error { + return db.Close() + }, nil +} + +func TestContent(t *testing.T) { + testsuite.ContentSuite(t, "metadata", createContentStore) } diff --git a/services/content/writer.go b/services/content/writer.go index eeb1fca3d..9a9c69a66 100644 --- a/services/content/writer.go +++ b/services/content/writer.go @@ -46,6 +46,7 @@ func (rw *remoteWriter) Status() (content.Status, error) { return content.Status{ Ref: rw.ref, Offset: resp.Offset, + Total: resp.Total, StartedAt: resp.StartedAt, UpdatedAt: resp.UpdatedAt, }, nil From dee8dc2cda2cc1ebb1b100262dac166733c3bffc Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Aug 2017 22:34:05 -0700 Subject: [PATCH 2/3] Add support for content labels on commit Add commit options which allow for setting labels on commit. Prevents potential race between garbage collector reading labels after commit and labels getting set. Signed-off-by: Derek McGowan --- api/services/content/v1/content.pb.go | 291 ++++++++++++++++++++------ api/services/content/v1/content.proto | 3 + content/content.go | 12 +- content/local/writer.go | 2 +- content/testsuite/testsuite.go | 70 +++++++ content_test.go | 1 + metadata/content.go | 15 +- metadata/content_test.go | 1 + remotes/docker/pusher.go | 2 +- services/content/service.go | 6 +- services/content/writer.go | 9 +- 11 files changed, 336 insertions(+), 76 deletions(-) diff --git a/api/services/content/v1/content.pb.go b/api/services/content/v1/content.pb.go index 86667e1b2..40cfc66d1 100644 --- a/api/services/content/v1/content.pb.go +++ b/api/services/content/v1/content.pb.go @@ -321,6 +321,8 @@ type WriteContentRequest struct { // If this is empty and the message is not a commit, a response will be // returned with the current write state. Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + // Labels are arbitrary data to set on commit. + Labels map[string]string `protobuf:"bytes,7,rep,name=labels" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *WriteContentRequest) Reset() { *m = WriteContentRequest{} } @@ -1445,6 +1447,23 @@ func (m *WriteContentRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintContent(dAtA, i, uint64(len(m.Data))) i += copy(dAtA[i:], m.Data) } + if len(m.Labels) > 0 { + for k, _ := range m.Labels { + dAtA[i] = 0x3a + i++ + v := m.Labels[k] + mapSize := 1 + len(k) + sovContent(uint64(len(k))) + 1 + len(v) + sovContent(uint64(len(v))) + i = encodeVarintContent(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintContent(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x12 + i++ + i = encodeVarintContent(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } return i, nil } @@ -1772,6 +1791,14 @@ func (m *WriteContentRequest) Size() (n int) { if l > 0 { n += 1 + l + sovContent(uint64(l)) } + if len(m.Labels) > 0 { + for k, v := range m.Labels { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovContent(uint64(len(k))) + 1 + len(v) + sovContent(uint64(len(v))) + n += mapEntrySize + 1 + sovContent(uint64(mapEntrySize)) + } + } return n } @@ -1998,6 +2025,16 @@ func (this *WriteContentRequest) String() string { if this == nil { return "nil" } + keysForLabels := make([]string, 0, len(this.Labels)) + for k, _ := range this.Labels { + keysForLabels = append(keysForLabels, k) + } + github_com_gogo_protobuf_sortkeys.Strings(keysForLabels) + mapStringForLabels := "map[string]string{" + for _, k := range keysForLabels { + mapStringForLabels += fmt.Sprintf("%v: %v,", k, this.Labels[k]) + } + mapStringForLabels += "}" s := strings.Join([]string{`&WriteContentRequest{`, `Action:` + fmt.Sprintf("%v", this.Action) + `,`, `Ref:` + fmt.Sprintf("%v", this.Ref) + `,`, @@ -2005,6 +2042,7 @@ func (this *WriteContentRequest) String() string { `Expected:` + fmt.Sprintf("%v", this.Expected) + `,`, `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, `Data:` + fmt.Sprintf("%v", this.Data) + `,`, + `Labels:` + mapStringForLabels + `,`, `}`, }, "") return s @@ -3827,6 +3865,122 @@ func (m *WriteContentRequest) Unmarshal(dAtA []byte) error { m.Data = []byte{} } iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Labels", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthContent + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthContent + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.Labels == nil { + m.Labels = make(map[string]string) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowContent + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthContent + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue := string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + m.Labels[mapkey] = mapvalue + } else { + var mapvalue string + m.Labels[mapkey] = mapvalue + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipContent(dAtA[iNdEx:]) @@ -4233,72 +4387,73 @@ func init() { } var fileDescriptorContent = []byte{ - // 1065 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0xf7, 0x78, 0xed, 0x4d, 0xfc, 0x9c, 0x16, 0x33, 0x36, 0x95, 0xb5, 0x08, 0x7b, 0xbb, 0x42, - 0xc8, 0x6a, 0xc9, 0x3a, 0x75, 0x72, 0x00, 0x2a, 0x21, 0x1c, 0x37, 0x55, 0x83, 0x9a, 0x82, 0xb6, - 0x2e, 0x15, 0xbd, 0x94, 0xb5, 0x3d, 0x36, 0xab, 0xd8, 0x5e, 0x77, 0x77, 0x6c, 0x11, 0x4e, 0x5c, - 0x90, 0x50, 0xd4, 0x03, 0x5f, 0x20, 0x17, 0xe0, 0x53, 0x70, 0xe0, 0x9c, 0x23, 0x47, 0xc4, 0xa1, - 0x25, 0xf9, 0x0e, 0x5c, 0x11, 0x9a, 0x3f, 0x6b, 0xaf, 0xed, 0x04, 0xff, 0x89, 0x39, 0xe5, 0xcd, - 0xf8, 0xfd, 0xde, 0xbc, 0xf7, 0x7b, 0xff, 0x36, 0x70, 0xbf, 0xe5, 0xd0, 0xaf, 0xfb, 0x35, 0xb3, - 0xee, 0x76, 0x8a, 0x75, 0xb7, 0x4b, 0x6d, 0xa7, 0x4b, 0xbc, 0x46, 0x58, 0xb4, 0x7b, 0x4e, 0xd1, - 0x27, 0xde, 0xc0, 0xa9, 0x13, 0x9f, 0xdf, 0x93, 0x2e, 0x2d, 0x0e, 0xee, 0x04, 0xa2, 0xd9, 0xf3, - 0x5c, 0xea, 0xe2, 0xdc, 0x08, 0x61, 0x06, 0xda, 0x66, 0xa0, 0x32, 0xb8, 0xa3, 0x65, 0x5a, 0x6e, - 0xcb, 0xe5, 0xaa, 0x45, 0x26, 0x09, 0x94, 0xa6, 0xb7, 0x5c, 0xb7, 0xd5, 0x26, 0x45, 0x7e, 0xaa, - 0xf5, 0x9b, 0xc5, 0xa6, 0x43, 0xda, 0x8d, 0xe7, 0x1d, 0xdb, 0x3f, 0x94, 0x1a, 0xf9, 0x49, 0x0d, - 0xea, 0x74, 0x88, 0x4f, 0xed, 0x4e, 0x4f, 0x2a, 0xbc, 0x3d, 0xa9, 0x40, 0x3a, 0x3d, 0x7a, 0x24, - 0x7e, 0x34, 0xfe, 0x8e, 0x42, 0x6c, 0xbf, 0xdb, 0x74, 0xf1, 0xa7, 0xa0, 0x36, 0x9c, 0x16, 0xf1, - 0x69, 0x16, 0xe9, 0xa8, 0x90, 0xd8, 0x2d, 0x9d, 0xbe, 0xca, 0x47, 0xfe, 0x7c, 0x95, 0xbf, 0x15, - 0x0a, 0xdf, 0xed, 0x91, 0xee, 0x30, 0x0a, 0xbf, 0xd8, 0x72, 0x37, 0x05, 0xc4, 0xbc, 0xc7, 0xff, - 0x58, 0xd2, 0x02, 0xc6, 0x10, 0xf3, 0x9d, 0x6f, 0x49, 0x36, 0xaa, 0xa3, 0x82, 0x62, 0x71, 0x19, - 0x57, 0x00, 0xea, 0x1e, 0xb1, 0x29, 0x69, 0x3c, 0xb7, 0x69, 0x56, 0xd1, 0x51, 0x21, 0x59, 0xd2, - 0x4c, 0xe1, 0x9a, 0x19, 0xb8, 0x66, 0x56, 0x03, 0xdf, 0x77, 0xd7, 0xd9, 0xfb, 0x3f, 0xbe, 0xce, - 0x23, 0x2b, 0x21, 0x71, 0x65, 0xca, 0x8c, 0xf4, 0x7b, 0x8d, 0xc0, 0x48, 0x6c, 0x11, 0x23, 0x12, - 0x57, 0xa6, 0xf8, 0x01, 0xa8, 0x6d, 0xbb, 0x46, 0xda, 0x7e, 0x36, 0xae, 0x2b, 0x85, 0x64, 0x69, - 0xcb, 0xfc, 0xef, 0xcc, 0x98, 0x8c, 0x1f, 0xf3, 0x21, 0x87, 0xec, 0x75, 0xa9, 0x77, 0x64, 0x49, - 0xbc, 0xf6, 0x21, 0x24, 0x43, 0xd7, 0x38, 0x05, 0xca, 0x21, 0x39, 0x12, 0xfc, 0x59, 0x4c, 0xc4, - 0x19, 0x88, 0x0f, 0xec, 0x76, 0x5f, 0x30, 0x91, 0xb0, 0xc4, 0xe1, 0xa3, 0xe8, 0x07, 0xc8, 0xf8, - 0x12, 0x92, 0xcc, 0xac, 0x45, 0x5e, 0xf4, 0x19, 0x63, 0x2b, 0x64, 0xdf, 0x78, 0x04, 0x1b, 0xc2, - 0xb4, 0xdf, 0x73, 0xbb, 0x3e, 0xc1, 0x1f, 0x43, 0xcc, 0xe9, 0x36, 0x5d, 0x6e, 0x39, 0x59, 0x7a, - 0x77, 0x9e, 0x68, 0x77, 0x63, 0xec, 0x7d, 0x8b, 0xe3, 0x8c, 0x97, 0x08, 0xae, 0x3d, 0xe1, 0xec, - 0x05, 0xde, 0x5e, 0xd1, 0x22, 0xbe, 0x0b, 0x49, 0x91, 0x0e, 0x5e, 0xc7, 0x9c, 0x9c, 0x8b, 0xf2, - 0x78, 0x9f, 0x95, 0xfa, 0x81, 0xed, 0x1f, 0x5a, 0x32, 0xeb, 0x4c, 0x36, 0x3e, 0x87, 0xeb, 0x81, - 0x37, 0x2b, 0x0a, 0xd0, 0x04, 0xfc, 0xd0, 0xf1, 0x69, 0x45, 0xa8, 0x04, 0x41, 0x66, 0x61, 0xad, - 0xe9, 0xb4, 0x29, 0xf1, 0xfc, 0x2c, 0xd2, 0x95, 0x42, 0xc2, 0x0a, 0x8e, 0xc6, 0x13, 0x48, 0x8f, - 0xe9, 0x4f, 0xb9, 0xa1, 0x2c, 0xe5, 0x46, 0x0d, 0x32, 0xf7, 0x48, 0x9b, 0x50, 0x32, 0xe1, 0xc8, - 0x2a, 0x6b, 0xe3, 0x25, 0x02, 0x6c, 0x11, 0xbb, 0xf1, 0xff, 0x3d, 0x81, 0x6f, 0x80, 0xea, 0x36, - 0x9b, 0x3e, 0xa1, 0xb2, 0xfd, 0xe5, 0x69, 0x38, 0x14, 0x94, 0xd1, 0x50, 0x30, 0xca, 0x90, 0x1e, - 0xf3, 0x46, 0x32, 0x39, 0x32, 0x81, 0x26, 0x4d, 0x34, 0x6c, 0x6a, 0x73, 0xc3, 0x1b, 0x16, 0x97, - 0x8d, 0x9f, 0xa2, 0xa0, 0x3e, 0xa6, 0x36, 0xed, 0xfb, 0x6c, 0x3a, 0xf8, 0xd4, 0xf6, 0xe4, 0x74, - 0x40, 0x8b, 0x4c, 0x07, 0x89, 0x9b, 0x1a, 0x31, 0xd1, 0xe5, 0x46, 0x4c, 0x0a, 0x14, 0x8f, 0x34, - 0x79, 0xa8, 0x09, 0x8b, 0x89, 0xa1, 0x90, 0x62, 0x63, 0x21, 0x65, 0x20, 0x4e, 0x5d, 0x6a, 0xb7, - 0xb3, 0x71, 0x7e, 0x2d, 0x0e, 0xf8, 0x11, 0xac, 0x93, 0x6f, 0x7a, 0xa4, 0x4e, 0x49, 0x23, 0xab, - 0x2e, 0x9d, 0x91, 0xa1, 0x0d, 0xe3, 0x26, 0x5c, 0x13, 0x1c, 0x05, 0x09, 0x97, 0x0e, 0xa2, 0xa1, - 0x83, 0xac, 0xad, 0x02, 0x95, 0x61, 0x3d, 0xab, 0x3e, 0xbf, 0x91, 0x54, 0xbe, 0x37, 0xab, 0xa2, - 0x25, 0x5e, 0xa2, 0x8c, 0xa2, 0x68, 0x13, 0x71, 0x4b, 0xfc, 0xd9, 0x7d, 0xf5, 0x15, 0x64, 0xc6, - 0x01, 0xd2, 0x91, 0x07, 0xb0, 0xee, 0xcb, 0x3b, 0xd9, 0x5c, 0x73, 0xba, 0x22, 0xdb, 0x6b, 0x88, - 0x36, 0xfe, 0x41, 0x90, 0x7e, 0xea, 0x39, 0x53, 0x2d, 0x56, 0x01, 0xd5, 0xae, 0x53, 0xc7, 0xed, - 0xf2, 0x50, 0xaf, 0x97, 0x6e, 0xcf, 0xb2, 0xcf, 0x8d, 0x94, 0x39, 0xc4, 0x92, 0xd0, 0x80, 0xd3, - 0xe8, 0x28, 0xe9, 0xc3, 0xe4, 0x2a, 0x97, 0x25, 0x37, 0x76, 0xf5, 0xe4, 0x86, 0x4a, 0x2b, 0x7e, - 0x61, 0xb7, 0xa8, 0xa1, 0x6e, 0x79, 0x1d, 0x85, 0xcc, 0x38, 0x01, 0x92, 0xe3, 0x95, 0x30, 0x30, - 0xde, 0x80, 0xd1, 0x55, 0x34, 0xa0, 0xb2, 0x5c, 0x03, 0x2e, 0xd6, 0x6e, 0xa3, 0xf1, 0xa7, 0x5e, - 0x79, 0xc2, 0xea, 0xb0, 0x51, 0xae, 0xb9, 0x1e, 0xbd, 0xb4, 0xd3, 0x6e, 0x7d, 0x8f, 0x20, 0x19, - 0x62, 0x0f, 0xbf, 0x03, 0xb1, 0xc7, 0xd5, 0x72, 0x35, 0x15, 0xd1, 0xd2, 0xc7, 0x27, 0xfa, 0x1b, - 0xa1, 0x9f, 0x58, 0x15, 0xe3, 0x3c, 0xc4, 0x9f, 0x5a, 0xfb, 0xd5, 0xbd, 0x14, 0xd2, 0x32, 0xc7, - 0x27, 0x7a, 0x2a, 0xf4, 0x3b, 0x17, 0xf1, 0x4d, 0x50, 0x2b, 0x9f, 0x1d, 0x1c, 0xec, 0x57, 0x53, - 0x51, 0xed, 0xad, 0xe3, 0x13, 0xfd, 0xcd, 0x90, 0x46, 0xc5, 0xed, 0x74, 0x1c, 0xaa, 0xa5, 0x7f, - 0xf8, 0x39, 0x17, 0xf9, 0xf5, 0x97, 0x5c, 0xf8, 0xdd, 0xd2, 0x6f, 0x6b, 0xb0, 0x26, 0xcb, 0x00, - 0xdb, 0xf2, 0x2b, 0xf0, 0xf6, 0x3c, 0x5b, 0x4b, 0x86, 0xa6, 0xbd, 0x3f, 0x9f, 0xb2, 0xac, 0xb0, - 0x16, 0xa8, 0x62, 0x6f, 0xe3, 0xcd, 0x59, 0xb8, 0xb1, 0xaf, 0x0d, 0xcd, 0x9c, 0x57, 0x5d, 0x3e, - 0xf4, 0x02, 0x62, 0x6c, 0x8c, 0xe0, 0xd2, 0x2c, 0xdc, 0xf4, 0xd2, 0xd7, 0xb6, 0x17, 0xc2, 0x88, - 0x07, 0xb7, 0x10, 0xfe, 0x02, 0x54, 0xb1, 0xba, 0xf1, 0xce, 0x2c, 0x03, 0x17, 0xad, 0x78, 0xed, - 0xc6, 0x54, 0x7d, 0xef, 0xb1, 0x6f, 0x74, 0x16, 0x0a, 0xdb, 0x8f, 0xb3, 0x43, 0x99, 0xde, 0xe9, - 0xb3, 0x43, 0xb9, 0x60, 0xf3, 0x6e, 0x21, 0x96, 0x26, 0xb9, 0x4e, 0x37, 0xe7, 0x9c, 0xf7, 0xf3, - 0xa6, 0x69, 0x62, 0xbd, 0x1c, 0xc1, 0x46, 0x78, 0xda, 0xe3, 0xb9, 0xa8, 0x9f, 0x58, 0x26, 0xda, - 0xce, 0x62, 0x20, 0xf9, 0xf4, 0x00, 0xe2, 0xa2, 0x75, 0xb6, 0xe7, 0x9a, 0x72, 0x13, 0xc4, 0xee, - 0x2c, 0x06, 0x12, 0x6f, 0x16, 0xd0, 0x16, 0xc2, 0x07, 0x10, 0xe7, 0xb3, 0x01, 0xcf, 0xec, 0x9c, - 0xf0, 0x08, 0xb9, 0xac, 0x3a, 0x76, 0x9f, 0x9d, 0x9e, 0xe5, 0x22, 0x7f, 0x9c, 0xe5, 0x22, 0xdf, - 0x9d, 0xe7, 0xd0, 0xe9, 0x79, 0x0e, 0xfd, 0x7e, 0x9e, 0x43, 0x7f, 0x9d, 0xe7, 0xd0, 0xb3, 0x4f, - 0x96, 0xfd, 0x9f, 0xf5, 0xae, 0x14, 0x6b, 0x2a, 0x7f, 0x6b, 0xfb, 0xdf, 0x00, 0x00, 0x00, 0xff, - 0xff, 0x28, 0x28, 0x53, 0x76, 0xfe, 0x0e, 0x00, 0x00, + // 1079 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0xcd, 0x6f, 0x1b, 0x45, + 0x14, 0xcf, 0x78, 0xed, 0x4d, 0xf2, 0x9c, 0x16, 0x33, 0x31, 0x95, 0xb5, 0x08, 0x7b, 0xbb, 0x42, + 0xc8, 0x6a, 0xc9, 0x3a, 0x75, 0x7a, 0x00, 0x2a, 0x01, 0x8e, 0x9b, 0xaa, 0x41, 0x4d, 0x41, 0x5b, + 0x97, 0x88, 0x5e, 0xca, 0xda, 0x1e, 0x9b, 0x55, 0x6c, 0xaf, 0xbb, 0x33, 0xb6, 0x08, 0x27, 0x2e, + 0x48, 0x28, 0xea, 0x01, 0x71, 0xcf, 0x05, 0xf8, 0x2b, 0x38, 0x70, 0xce, 0x91, 0x23, 0xe2, 0xd0, + 0xd2, 0xfc, 0x0f, 0xdc, 0xd1, 0xcc, 0xce, 0xda, 0xeb, 0x8f, 0xb0, 0xb6, 0xe3, 0x9e, 0xfc, 0x66, + 0xf6, 0xfd, 0xde, 0xf7, 0xc7, 0x18, 0xee, 0x35, 0x1d, 0xf6, 0x4d, 0xaf, 0x6a, 0xd6, 0xdc, 0x76, + 0xa1, 0xe6, 0x76, 0x98, 0xed, 0x74, 0x88, 0x57, 0x0f, 0x93, 0x76, 0xd7, 0x29, 0x50, 0xe2, 0xf5, + 0x9d, 0x1a, 0xa1, 0xe2, 0x9e, 0x74, 0x58, 0xa1, 0x7f, 0x2b, 0x20, 0xcd, 0xae, 0xe7, 0x32, 0x17, + 0x67, 0x87, 0x08, 0x33, 0xe0, 0x36, 0x03, 0x96, 0xfe, 0x2d, 0x2d, 0xdd, 0x74, 0x9b, 0xae, 0x60, + 0x2d, 0x70, 0xca, 0x47, 0x69, 0x7a, 0xd3, 0x75, 0x9b, 0x2d, 0x52, 0x10, 0xa7, 0x6a, 0xaf, 0x51, + 0x68, 0x38, 0xa4, 0x55, 0x7f, 0xda, 0xb6, 0xe9, 0x91, 0xe4, 0xc8, 0x8d, 0x73, 0x30, 0xa7, 0x4d, + 0x28, 0xb3, 0xdb, 0x5d, 0xc9, 0xf0, 0xf6, 0x38, 0x03, 0x69, 0x77, 0xd9, 0xb1, 0xff, 0xd1, 0xf8, + 0x37, 0x06, 0xf1, 0xfd, 0x4e, 0xc3, 0xc5, 0x9f, 0x81, 0x5a, 0x77, 0x9a, 0x84, 0xb2, 0x0c, 0xd2, + 0x51, 0x7e, 0x7d, 0xb7, 0x78, 0xf6, 0x22, 0xb7, 0xf2, 0xf7, 0x8b, 0xdc, 0x8d, 0x90, 0xfb, 0x6e, + 0x97, 0x74, 0x06, 0x5e, 0xd0, 0x42, 0xd3, 0xdd, 0xf2, 0x21, 0xe6, 0x5d, 0xf1, 0x63, 0x49, 0x09, + 0x18, 0x43, 0x9c, 0x3a, 0xdf, 0x91, 0x4c, 0x4c, 0x47, 0x79, 0xc5, 0x12, 0x34, 0x2e, 0x03, 0xd4, + 0x3c, 0x62, 0x33, 0x52, 0x7f, 0x6a, 0xb3, 0x8c, 0xa2, 0xa3, 0x7c, 0xb2, 0xa8, 0x99, 0xbe, 0x69, + 0x66, 0x60, 0x9a, 0x59, 0x09, 0x6c, 0xdf, 0x5d, 0xe3, 0xfa, 0x7f, 0x7a, 0x99, 0x43, 0xd6, 0xba, + 0xc4, 0x95, 0x18, 0x17, 0xd2, 0xeb, 0xd6, 0x03, 0x21, 0xf1, 0x79, 0x84, 0x48, 0x5c, 0x89, 0xe1, + 0xfb, 0xa0, 0xb6, 0xec, 0x2a, 0x69, 0xd1, 0x4c, 0x42, 0x57, 0xf2, 0xc9, 0xe2, 0xb6, 0xf9, 0xff, + 0x99, 0x31, 0x79, 0x7c, 0xcc, 0x07, 0x02, 0xb2, 0xd7, 0x61, 0xde, 0xb1, 0x25, 0xf1, 0xda, 0x87, + 0x90, 0x0c, 0x5d, 0xe3, 0x14, 0x28, 0x47, 0xe4, 0xd8, 0x8f, 0x9f, 0xc5, 0x49, 0x9c, 0x86, 0x44, + 0xdf, 0x6e, 0xf5, 0xfc, 0x48, 0xac, 0x5b, 0xfe, 0xe1, 0xa3, 0xd8, 0x07, 0xc8, 0xf8, 0x0a, 0x92, + 0x5c, 0xac, 0x45, 0x9e, 0xf5, 0x78, 0xc4, 0x96, 0x18, 0x7d, 0xe3, 0x21, 0x6c, 0xf8, 0xa2, 0x69, + 0xd7, 0xed, 0x50, 0x82, 0x3f, 0x86, 0xb8, 0xd3, 0x69, 0xb8, 0x42, 0x72, 0xb2, 0xf8, 0xee, 0x2c, + 0xde, 0xee, 0xc6, 0xb9, 0x7e, 0x4b, 0xe0, 0x8c, 0xe7, 0x08, 0xae, 0x3c, 0x16, 0xd1, 0x0b, 0xac, + 0xbd, 0xa4, 0x44, 0x7c, 0x07, 0x92, 0x7e, 0x3a, 0x44, 0x1d, 0x8b, 0xe0, 0x4c, 0xcb, 0xe3, 0x3d, + 0x5e, 0xea, 0x07, 0x36, 0x3d, 0xb2, 0x64, 0xd6, 0x39, 0x6d, 0x7c, 0x01, 0x57, 0x03, 0x6b, 0x96, + 0xe4, 0xa0, 0x09, 0xf8, 0x81, 0x43, 0x59, 0xd9, 0x67, 0x09, 0x9c, 0xcc, 0xc0, 0x6a, 0xc3, 0x69, + 0x31, 0xe2, 0xd1, 0x0c, 0xd2, 0x95, 0xfc, 0xba, 0x15, 0x1c, 0x8d, 0xc7, 0xb0, 0x39, 0xc2, 0x3f, + 0x61, 0x86, 0xb2, 0x90, 0x19, 0x55, 0x48, 0xdf, 0x25, 0x2d, 0xc2, 0xc8, 0x98, 0x21, 0xcb, 0xac, + 0x8d, 0xe7, 0x08, 0xb0, 0x45, 0xec, 0xfa, 0xeb, 0x53, 0x81, 0xaf, 0x81, 0xea, 0x36, 0x1a, 0x94, + 0x30, 0xd9, 0xfe, 0xf2, 0x34, 0x18, 0x0a, 0xca, 0x70, 0x28, 0x18, 0x25, 0xd8, 0x1c, 0xb1, 0x46, + 0x46, 0x72, 0x28, 0x02, 0x8d, 0x8b, 0xa8, 0xdb, 0xcc, 0x16, 0x82, 0x37, 0x2c, 0x41, 0x1b, 0xbf, + 0xc4, 0x40, 0x7d, 0xc4, 0x6c, 0xd6, 0xa3, 0x7c, 0x3a, 0x50, 0x66, 0x7b, 0x72, 0x3a, 0xa0, 0x79, + 0xa6, 0x83, 0xc4, 0x4d, 0x8c, 0x98, 0xd8, 0x62, 0x23, 0x26, 0x05, 0x8a, 0x47, 0x1a, 0xc2, 0xd5, + 0x75, 0x8b, 0x93, 0x21, 0x97, 0xe2, 0x23, 0x2e, 0xa5, 0x21, 0xc1, 0x5c, 0x66, 0xb7, 0x32, 0x09, + 0x71, 0xed, 0x1f, 0xf0, 0x43, 0x58, 0x23, 0xdf, 0x76, 0x49, 0x8d, 0x91, 0x7a, 0x46, 0x5d, 0x38, + 0x23, 0x03, 0x19, 0xc6, 0x75, 0xb8, 0xe2, 0xc7, 0x28, 0x48, 0xb8, 0x34, 0x10, 0x0d, 0x0c, 0xe4, + 0x6d, 0x15, 0xb0, 0x0c, 0xea, 0x59, 0xa5, 0xe2, 0x46, 0x86, 0xf2, 0xbd, 0xa8, 0x8a, 0x96, 0x78, + 0x89, 0x32, 0x0a, 0x7e, 0x9b, 0xf8, 0xb7, 0x84, 0x46, 0xf7, 0xd5, 0xd7, 0x90, 0x1e, 0x05, 0x48, + 0x43, 0xee, 0xc3, 0x1a, 0x95, 0x77, 0xb2, 0xb9, 0x66, 0x34, 0x45, 0xb6, 0xd7, 0x00, 0x6d, 0xfc, + 0xac, 0xc0, 0xe6, 0xa1, 0xe7, 0x4c, 0xb4, 0x58, 0x19, 0x54, 0xbb, 0xc6, 0x1c, 0xb7, 0x23, 0x5c, + 0xbd, 0x5a, 0xbc, 0x19, 0x25, 0x5f, 0x08, 0x29, 0x09, 0x88, 0x25, 0xa1, 0x41, 0x4c, 0x63, 0xc3, + 0xa4, 0x0f, 0x92, 0xab, 0x5c, 0x94, 0xdc, 0xf8, 0xe5, 0x93, 0x1b, 0x2a, 0xad, 0xc4, 0xd4, 0x6e, + 0x51, 0x87, 0xdd, 0x82, 0x0f, 0x07, 0xbb, 0x6f, 0x55, 0x04, 0xf2, 0x93, 0x99, 0x1c, 0x1d, 0x8d, + 0xd6, 0xb2, 0x57, 0xe1, 0xcb, 0x18, 0xa4, 0x47, 0xd5, 0xc8, 0xbc, 0x2f, 0x25, 0x2b, 0xa3, 0x43, + 0x21, 0xb6, 0x8c, 0xa1, 0xa0, 0x2c, 0x36, 0x14, 0xe6, 0x1b, 0x01, 0xc3, 0x91, 0xac, 0x5e, 0x7a, + 0xea, 0xeb, 0xb0, 0x51, 0xaa, 0xba, 0x1e, 0xbb, 0xb0, 0xfb, 0x6f, 0xfc, 0x80, 0x20, 0x19, 0x8a, + 0x1e, 0x7e, 0x07, 0xe2, 0x8f, 0x2a, 0xa5, 0x4a, 0x6a, 0x45, 0xdb, 0x3c, 0x39, 0xd5, 0xdf, 0x08, + 0x7d, 0xe2, 0x9d, 0x85, 0x73, 0x90, 0x38, 0xb4, 0xf6, 0x2b, 0x7b, 0x29, 0xa4, 0xa5, 0x4f, 0x4e, + 0xf5, 0x54, 0xe8, 0xbb, 0x20, 0xf1, 0x75, 0x50, 0xcb, 0x9f, 0x1f, 0x1c, 0xec, 0x57, 0x52, 0x31, + 0xed, 0xad, 0x93, 0x53, 0xfd, 0xcd, 0x10, 0x47, 0xd9, 0x6d, 0xb7, 0x1d, 0xa6, 0x6d, 0xfe, 0xf8, + 0x6b, 0x76, 0xe5, 0xf7, 0xdf, 0xb2, 0x61, 0xbd, 0xc5, 0x3f, 0x56, 0x61, 0x55, 0x96, 0x01, 0xb6, + 0xe5, 0xcb, 0xf4, 0xe6, 0x2c, 0x9b, 0x54, 0xba, 0xa6, 0xbd, 0x3f, 0x1b, 0xb3, 0xac, 0xb0, 0x26, + 0xa8, 0xfe, 0x5b, 0x02, 0x6f, 0x45, 0xe1, 0x46, 0x5e, 0x40, 0x9a, 0x39, 0x2b, 0xbb, 0x54, 0xf4, + 0x0c, 0xe2, 0x7c, 0xb4, 0xe1, 0x62, 0x14, 0x6e, 0xf2, 0x21, 0xa2, 0xed, 0xcc, 0x85, 0xf1, 0x15, + 0x6e, 0x23, 0xfc, 0x25, 0xa8, 0xfe, 0x73, 0x02, 0xdf, 0x8e, 0x12, 0x30, 0xed, 0xd9, 0xa1, 0x5d, + 0x9b, 0xa8, 0xef, 0x3d, 0xfe, 0xbf, 0x81, 0xbb, 0xc2, 0x77, 0x76, 0xb4, 0x2b, 0x93, 0xef, 0x8c, + 0x68, 0x57, 0xa6, 0xbc, 0x06, 0xb6, 0x11, 0x4f, 0x93, 0x5c, 0xf1, 0x5b, 0x33, 0xee, 0xa0, 0x59, + 0xd3, 0x34, 0xb6, 0xf2, 0x8e, 0x61, 0x23, 0xbc, 0x81, 0xf0, 0x4c, 0xa1, 0x1f, 0x5b, 0x70, 0xda, + 0xed, 0xf9, 0x40, 0x52, 0x75, 0x1f, 0x12, 0x7e, 0xeb, 0xec, 0x2c, 0x30, 0x92, 0xa3, 0x75, 0x4e, + 0x1b, 0xb0, 0x79, 0xb4, 0x8d, 0xf0, 0x01, 0x24, 0xc4, 0x6c, 0xc0, 0x91, 0x9d, 0x13, 0x1e, 0x21, + 0x17, 0x55, 0xc7, 0xee, 0x93, 0xb3, 0x57, 0xd9, 0x95, 0xbf, 0x5e, 0x65, 0x57, 0xbe, 0x3f, 0xcf, + 0xa2, 0xb3, 0xf3, 0x2c, 0xfa, 0xf3, 0x3c, 0x8b, 0xfe, 0x39, 0xcf, 0xa2, 0x27, 0x9f, 0x2e, 0xfa, + 0x3f, 0xfa, 0x8e, 0x24, 0xab, 0xaa, 0xd0, 0xb5, 0xf3, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xbf, + 0xc1, 0xae, 0xf1, 0x92, 0x0f, 0x00, 0x00, } diff --git a/api/services/content/v1/content.proto b/api/services/content/v1/content.proto index daa0a2d93..9538408bd 100644 --- a/api/services/content/v1/content.proto +++ b/api/services/content/v1/content.proto @@ -269,6 +269,9 @@ message WriteContentRequest { // If this is empty and the message is not a commit, a response will be // returned with the current write state. bytes data = 6; + + // Labels are arbitrary data to set on commit. + map labels = 7; } // WriteContentResponse is returned on the culmination of a write call. diff --git a/content/content.go b/content/content.go index 58a2ff436..d7a51a826 100644 --- a/content/content.go +++ b/content/content.go @@ -90,7 +90,7 @@ type Writer interface { // Commit commits the blob (but no roll-back is guaranteed on an error). // size and expected can be zero-value when unknown. - Commit(size int64, expected digest.Digest) error + Commit(size int64, expected digest.Digest, opts ...Opt) error // Status returns the current state of write Status() (Status, error) @@ -107,3 +107,13 @@ type Store interface { IngestManager Ingester } + +// Opt is used to alter the mutable properties of content +type Opt func(*Info) error + +func WithLabels(labels map[string]string) Opt { + return func(info *Info) error { + info.Labels = labels + return nil + } +} diff --git a/content/local/writer.go b/content/local/writer.go index dfa8e1290..a7e6e25e4 100644 --- a/content/local/writer.go +++ b/content/local/writer.go @@ -54,7 +54,7 @@ func (w *writer) Write(p []byte) (n int, err error) { return n, err } -func (w *writer) Commit(size int64, expected digest.Digest) error { +func (w *writer) Commit(size int64, expected digest.Digest, opts ...content.Opt) error { if err := w.fp.Sync(); err != nil { return errors.Wrap(err, "sync failed") } diff --git a/content/testsuite/testsuite.go b/content/testsuite/testsuite.go index afeb4d918..7c45fa01a 100644 --- a/content/testsuite/testsuite.go +++ b/content/testsuite/testsuite.go @@ -23,6 +23,13 @@ func ContentSuite(t *testing.T, name string, storeFn func(ctx context.Context, r t.Run("UploadStatus", makeTest(t, name, storeFn, checkUploadStatus)) } +// ContentLabelSuite runs a test suite for the content store supporting +// labels. +// TODO: Merge this with ContentSuite once all content stores support labels +func ContentLabelSuite(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func() error, error)) { + t.Run("Labels", makeTest(t, name, storeFn, checkLabels)) +} + func makeTest(t *testing.T, name string, storeFn func(ctx context.Context, root string) (content.Store, func() error, error), fn func(ctx context.Context, t *testing.T, cs content.Store)) func(t *testing.T) { return func(t *testing.T) { ctx := namespaces.WithNamespace(context.Background(), name) @@ -204,6 +211,69 @@ func checkUploadStatus(ctx context.Context, t *testing.T, cs content.Store) { } } +func checkLabels(ctx context.Context, t *testing.T, cs content.Store) { + c1, d1 := createContent(256, 1) + + w1, err := cs.Writer(ctx, "c1", 256, d1) + if err != nil { + t.Fatal(err) + } + + if _, err := w1.Write(c1); err != nil { + t.Fatalf("Failed to write: %+v", err) + } + + labels := map[string]string{ + "k1": "v1", + "k2": "v2", + } + + preCommit := time.Now() + if err := w1.Commit(0, "", content.WithLabels(labels)); err != nil { + t.Fatalf("Commit failed: %+v", err) + } + postCommit := time.Now() + + info := content.Info{ + Digest: d1, + Size: 256, + Labels: labels, + } + + if err := checkInfo(ctx, cs, d1, info, preCommit, postCommit, preCommit, postCommit); err != nil { + t.Fatalf("Check info failed: %+v", err) + } + + labels["k1"] = "newvalue" + delete(labels, "k2") + labels["k3"] = "v3" + + info.Labels = labels + preUpdate := time.Now() + if _, err := cs.Update(ctx, info); err != nil { + t.Fatalf("Update failed: %+v", err) + } + postUpdate := time.Now() + + if err := checkInfo(ctx, cs, d1, info, preCommit, postCommit, preUpdate, postUpdate); err != nil { + t.Fatalf("Check info failed: %+v", err) + } + + info.Labels = map[string]string{ + "k1": "v1", + } + preUpdate = time.Now() + if _, err := cs.Update(ctx, info, "labels.k3", "labels.k1"); err != nil { + t.Fatalf("Update failed: %+v", err) + } + postUpdate = time.Now() + + if err := checkInfo(ctx, cs, d1, info, preCommit, postCommit, preUpdate, postUpdate); err != nil { + t.Fatalf("Check info failed: %+v", err) + } + +} + func checkStatus(w content.Writer, expected content.Status, d digest.Digest, preStart, postStart, preUpdate, postUpdate time.Time) error { st, err := w.Status() if err != nil { diff --git a/content_test.go b/content_test.go index 6fc7818ae..2a0c68630 100644 --- a/content_test.go +++ b/content_test.go @@ -39,4 +39,5 @@ func TestContentClient(t *testing.T) { t.Skip() } testsuite.ContentSuite(t, "ContentClient", newContentStore) + testsuite.ContentLabelSuite(t, "ContentClient", newContentStore) } diff --git a/metadata/content.go b/metadata/content.go index c0b3fc795..91dd33c22 100644 --- a/metadata/content.go +++ b/metadata/content.go @@ -352,7 +352,7 @@ type namespacedWriter struct { db *bolt.DB } -func (nw *namespacedWriter) Commit(size int64, expected digest.Digest) error { +func (nw *namespacedWriter) Commit(size int64, expected digest.Digest, opts ...content.Opt) error { return nw.db.Update(func(tx *bolt.Tx) error { bkt := getIngestBucket(tx, nw.namespace) if bkt != nil { @@ -360,11 +360,17 @@ func (nw *namespacedWriter) Commit(size int64, expected digest.Digest) error { return err } } - return nw.commit(tx, size, expected) + return nw.commit(tx, size, expected, opts...) }) } -func (nw *namespacedWriter) commit(tx *bolt.Tx, size int64, expected digest.Digest) error { +func (nw *namespacedWriter) commit(tx *bolt.Tx, size int64, expected digest.Digest, opts ...content.Opt) error { + var base content.Info + for _, opt := range opts { + if err := opt(&base); err != nil { + return err + } + } status, err := nw.Writer.Status() if err != nil { return err @@ -400,6 +406,9 @@ func (nw *namespacedWriter) commit(tx *bolt.Tx, size int64, expected digest.Dige if err := boltutil.WriteTimestamps(bkt, commitTime, commitTime); err != nil { return err } + if err := boltutil.WriteLabels(bkt, base.Labels); err != nil { + return err + } if err := bkt.Put(bucketKeySize, sizeEncoded); err != nil { return err } diff --git a/metadata/content_test.go b/metadata/content_test.go index 093736833..8d23c33f9 100644 --- a/metadata/content_test.go +++ b/metadata/content_test.go @@ -30,4 +30,5 @@ func createContentStore(ctx context.Context, root string) (content.Store, func() func TestContent(t *testing.T) { testsuite.ContentSuite(t, "metadata", createContentStore) + testsuite.ContentLabelSuite(t, "metadata", createContentStore) } diff --git a/remotes/docker/pusher.go b/remotes/docker/pusher.go index bc9c42e2a..f90eace51 100644 --- a/remotes/docker/pusher.go +++ b/remotes/docker/pusher.go @@ -215,7 +215,7 @@ func (pw *pushWriter) Digest() digest.Digest { return pw.expected } -func (pw *pushWriter) Commit(size int64, expected digest.Digest) error { +func (pw *pushWriter) Commit(size int64, expected digest.Digest, opts ...content.Opt) error { // Check whether read has already thrown an error if _, err := pw.pipe.Write([]byte{}); err != nil && err != io.ErrClosedPipe { return errors.Wrap(err, "pipe error before commit") diff --git a/services/content/service.go b/services/content/service.go index f75f89071..2ff4b908a 100644 --- a/services/content/service.go +++ b/services/content/service.go @@ -420,7 +420,11 @@ func (s *Service) Write(session api.Content_WriteServer) (err error) { } if req.Action == api.WriteActionCommit { - if err := wr.Commit(total, expected); err != nil { + var opts []content.Opt + if req.Labels != nil { + opts = append(opts, content.WithLabels(req.Labels)) + } + if err := wr.Commit(total, expected, opts...); err != nil { return err } } diff --git a/services/content/writer.go b/services/content/writer.go index 9a9c69a66..ed94e4167 100644 --- a/services/content/writer.go +++ b/services/content/writer.go @@ -80,12 +80,19 @@ func (rw *remoteWriter) Write(p []byte) (n int, err error) { return } -func (rw *remoteWriter) Commit(size int64, expected digest.Digest) error { +func (rw *remoteWriter) Commit(size int64, expected digest.Digest, opts ...content.Opt) error { + var base content.Info + for _, opt := range opts { + if err := opt(&base); err != nil { + return err + } + } resp, err := rw.send(&contentapi.WriteContentRequest{ Action: contentapi.WriteActionCommit, Total: size, Offset: rw.offset, Expected: expected, + Labels: base.Labels, }) if err != nil { return errdefs.FromGRPC(err) From da34812db6890f80c7fac6f6021e27452fa1162a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 9 Aug 2017 22:35:49 -0700 Subject: [PATCH 3/3] Update differ to support compressed archives Differ is updated to set a label for the uncompressed hash of compressed content. This allows compressed blobs to be used and looked up for their uncompressed hashes. Uses commit options to set labels. Updates rootfs default to create compressed archives. Signed-off-by: Derek McGowan --- differ/differ.go | 48 +++++++++++++++++++++++++++++++----------------- rootfs/diff.go | 2 +- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/differ/differ.go b/differ/differ.go index 9cd525399..ad603a366 100644 --- a/differ/differ.go +++ b/differ/differ.go @@ -102,7 +102,17 @@ func (s *BaseDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts [] } func (s *BaseDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount, media, ref string) (ocispec.Descriptor, error) { - + var isCompressed bool + switch media { + case ocispec.MediaTypeImageLayer: + case ocispec.MediaTypeImageLayerGzip: + isCompressed = true + case "": + media = ocispec.MediaTypeImageLayerGzip + isCompressed = true + default: + return emptyDesc, errors.Errorf("unsupported diff media type: %v", media) + } aDir, err := ioutil.TempDir("", "left-") if err != nil { return emptyDesc, errors.Wrap(err, "failed to create temporary directory") @@ -130,25 +140,29 @@ func (s *BaseDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount, m return emptyDesc, errors.Wrap(err, "failed to open writer") } - // TODO: Validate media type - - // TODO: Support compressed media types (link compressed to uncompressed) - //dgstr := digest.SHA256.Digester() - //wc := &writeCounter{} - //compressed, err := compression.CompressStream(cw, compression.Gzip) - //if err != nil { - // return nil, errors.Wrap(err, "failed to get compressed stream") - //} - //err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash(), wc), lowerDir, upperDir) - //compressed.Close() - - err = archive.WriteDiff(ctx, cw, aDir, bDir) - if err != nil { - return emptyDesc, errors.Wrap(err, "failed to write diff") + var opts []content.Opt + if isCompressed { + dgstr := digest.SHA256.Digester() + compressed, err := compression.CompressStream(cw, compression.Gzip) + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to get compressed stream") + } + err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash()), aDir, bDir) + compressed.Close() + if err != nil { + return emptyDesc, errors.Wrap(err, "failed to write compressed diff") + } + opts = append(opts, content.WithLabels(map[string]string{ + "containerd.io/uncompressed": dgstr.Digest().String(), + })) + } else { + if err = archive.WriteDiff(ctx, cw, aDir, bDir); err != nil { + return emptyDesc, errors.Wrap(err, "failed to write diff") + } } dgst := cw.Digest() - if err := cw.Commit(0, dgst); err != nil { + if err := cw.Commit(0, dgst, opts...); err != nil { return emptyDesc, errors.Wrap(err, "failed to commit") } diff --git a/rootfs/diff.go b/rootfs/diff.go index c83ab4f1f..1a16fb9d9 100644 --- a/rootfs/diff.go +++ b/rootfs/diff.go @@ -49,5 +49,5 @@ func Diff(ctx context.Context, snapshotID, contentRef string, sn snapshot.Snapsh defer sn.Remove(ctx, lowerKey) } - return md.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, contentRef) + return md.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayerGzip, contentRef) }