From c6703d4c7622e1debc1451b47713ae93f167c02e Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Mon, 11 Feb 2019 11:46:14 -0500 Subject: [PATCH 1/6] Add missing annotations map to Descriptor for gRPC transfer Add the annotations map to the gRPC Descriptor message. Signed-off-by: Stefan Berger Signed-off-by: Brandon Lum Signed-off-by: Harshal Patil --- api/next.pb.txt | 28 ++++++ api/types/descriptor.pb.go | 189 ++++++++++++++++++++++++++++++++++--- api/types/descriptor.proto | 1 + 3 files changed, 204 insertions(+), 14 deletions(-) diff --git a/api/next.pb.txt b/api/next.pb.txt index 288556afa..b5eed3949 100755 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -1764,6 +1764,34 @@ file { type: TYPE_INT64 json_name: "size" } + field { + name: "annotations" + number: 5 + label: LABEL_REPEATED + type: TYPE_MESSAGE + type_name: ".containerd.types.Descriptor.AnnotationsEntry" + json_name: "annotations" + } + nested_type { + name: "AnnotationsEntry" + field { + name: "key" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "key" + } + field { + name: "value" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "value" + } + options { + map_entry: true + } + } } options { go_package: "github.com/containerd/containerd/api/types;types" diff --git a/api/types/descriptor.pb.go b/api/types/descriptor.pb.go index 93e88c0dc..4f5f6c61f 100644 --- a/api/types/descriptor.pb.go +++ b/api/types/descriptor.pb.go @@ -28,6 +28,7 @@ import github_com_opencontainers_go_digest "github.com/opencontainers/go-digest" import strings "strings" import reflect "reflect" +import sortkeys "github.com/gogo/protobuf/sortkeys" import io "io" @@ -48,9 +49,10 @@ const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package // oci descriptor found in a manifest. // See https://godoc.org/github.com/opencontainers/image-spec/specs-go/v1#Descriptor type Descriptor struct { - MediaType string `protobuf:"bytes,1,opt,name=media_type,json=mediaType,proto3" json:"media_type,omitempty"` - Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,2,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` - Size_ int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + MediaType string `protobuf:"bytes,1,opt,name=media_type,json=mediaType,proto3" json:"media_type,omitempty"` + Digest github_com_opencontainers_go_digest.Digest `protobuf:"bytes,2,opt,name=digest,proto3,customtype=github.com/opencontainers/go-digest.Digest" json:"digest"` + Size_ int64 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + Annotations map[string]string `protobuf:"bytes,5,rep,name=annotations" json:"annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (m *Descriptor) Reset() { *m = Descriptor{} } @@ -92,6 +94,23 @@ func (m *Descriptor) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintDescriptor(dAtA, i, uint64(m.Size_)) } + if len(m.Annotations) > 0 { + for k, _ := range m.Annotations { + dAtA[i] = 0x2a + i++ + v := m.Annotations[k] + mapSize := 1 + len(k) + sovDescriptor(uint64(len(k))) + 1 + len(v) + sovDescriptor(uint64(len(v))) + i = encodeVarintDescriptor(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintDescriptor(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x12 + i++ + i = encodeVarintDescriptor(dAtA, i, uint64(len(v))) + i += copy(dAtA[i:], v) + } + } return i, nil } @@ -118,6 +137,14 @@ func (m *Descriptor) Size() (n int) { if m.Size_ != 0 { n += 1 + sovDescriptor(uint64(m.Size_)) } + if len(m.Annotations) > 0 { + for k, v := range m.Annotations { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovDescriptor(uint64(len(k))) + 1 + len(v) + sovDescriptor(uint64(len(v))) + n += mapEntrySize + 1 + sovDescriptor(uint64(mapEntrySize)) + } + } return n } @@ -138,10 +165,21 @@ func (this *Descriptor) String() string { if this == nil { return "nil" } + keysForAnnotations := make([]string, 0, len(this.Annotations)) + for k, _ := range this.Annotations { + keysForAnnotations = append(keysForAnnotations, k) + } + sortkeys.Strings(keysForAnnotations) + mapStringForAnnotations := "map[string]string{" + for _, k := range keysForAnnotations { + mapStringForAnnotations += fmt.Sprintf("%v: %v,", k, this.Annotations[k]) + } + mapStringForAnnotations += "}" s := strings.Join([]string{`&Descriptor{`, `MediaType:` + fmt.Sprintf("%v", this.MediaType) + `,`, `Digest:` + fmt.Sprintf("%v", this.Digest) + `,`, `Size_:` + fmt.Sprintf("%v", this.Size_) + `,`, + `Annotations:` + mapStringForAnnotations + `,`, `}`, }, "") return s @@ -260,6 +298,124 @@ func (m *Descriptor) Unmarshal(dAtA []byte) error { break } } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Annotations", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDescriptor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthDescriptor + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Annotations == nil { + m.Annotations = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDescriptor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDescriptor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthDescriptor + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDescriptor + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthDescriptor + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := skipDescriptor(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthDescriptor + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Annotations[mapkey] = mapvalue + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipDescriptor(dAtA[iNdEx:]) @@ -391,20 +547,25 @@ func init() { } var fileDescriptorDescriptor = []byte{ - // 234 bytes of a gzipped FileDescriptorProto + // 311 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xb2, 0x4e, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xce, 0xcf, 0x2b, 0x49, 0xcc, 0xcc, 0x4b, 0x2d, 0x4a, 0x41, 0x66, 0x26, 0x16, 0x64, 0xea, 0x97, 0x54, 0x16, 0xa4, 0x16, 0xeb, 0xa7, 0xa4, 0x16, 0x27, 0x17, 0x65, 0x16, 0x94, 0xe4, 0x17, 0xe9, 0x15, 0x14, 0xe5, 0x97, 0xe4, 0x0b, 0x09, 0x20, 0x94, 0xe9, 0x81, 0x95, 0x48, 0x89, 0xa4, 0xe7, 0xa7, 0xe7, 0x83, 0x25, 0xf5, 0x41, 0x2c, 0x88, - 0x3a, 0xa5, 0x6e, 0x46, 0x2e, 0x2e, 0x17, 0xb8, 0x66, 0x21, 0x59, 0x2e, 0xae, 0xdc, 0xd4, 0x94, - 0xcc, 0xc4, 0x78, 0x90, 0x1e, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xce, 0x20, 0x4e, 0xb0, 0x48, 0x48, - 0x65, 0x41, 0xaa, 0x90, 0x17, 0x17, 0x5b, 0x4a, 0x66, 0x7a, 0x6a, 0x71, 0x89, 0x04, 0x13, 0x48, - 0xca, 0xc9, 0xe8, 0xc4, 0x3d, 0x79, 0x86, 0x5b, 0xf7, 0xe4, 0xb5, 0x90, 0x9c, 0x9a, 0x5f, 0x90, - 0x9a, 0x07, 0xb7, 0xbc, 0x58, 0x3f, 0x3d, 0x5f, 0x17, 0xa2, 0x45, 0xcf, 0x05, 0x4c, 0x05, 0x41, - 0x4d, 0x10, 0x12, 0xe2, 0x62, 0x29, 0xce, 0xac, 0x4a, 0x95, 0x60, 0x56, 0x60, 0xd4, 0x60, 0x0e, - 0x02, 0xb3, 0x9d, 0xbc, 0x4e, 0x3c, 0x94, 0x63, 0xb8, 0xf1, 0x50, 0x8e, 0xa1, 0xe1, 0x91, 0x1c, - 0xe3, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9, 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x18, 0x65, 0x40, - 0x7c, 0x60, 0x58, 0x83, 0xc9, 0x08, 0x86, 0x24, 0x36, 0xb0, 0x17, 0x8d, 0x01, 0x01, 0x00, 0x00, - 0xff, 0xff, 0xea, 0xac, 0x78, 0x9a, 0x49, 0x01, 0x00, 0x00, + 0x3a, 0xa5, 0x39, 0x4c, 0x5c, 0x5c, 0x2e, 0x70, 0xcd, 0x42, 0xb2, 0x5c, 0x5c, 0xb9, 0xa9, 0x29, + 0x99, 0x89, 0xf1, 0x20, 0x3d, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x41, 0x9c, 0x60, 0x91, 0x90, + 0xca, 0x82, 0x54, 0x21, 0x2f, 0x2e, 0xb6, 0x94, 0xcc, 0xf4, 0xd4, 0xe2, 0x12, 0x09, 0x26, 0x90, + 0x94, 0x93, 0xd1, 0x89, 0x7b, 0xf2, 0x0c, 0xb7, 0xee, 0xc9, 0x6b, 0x21, 0x39, 0x35, 0xbf, 0x20, + 0x35, 0x0f, 0x6e, 0x79, 0xb1, 0x7e, 0x7a, 0xbe, 0x2e, 0x44, 0x8b, 0x9e, 0x0b, 0x98, 0x0a, 0x82, + 0x9a, 0x20, 0x24, 0xc4, 0xc5, 0x52, 0x9c, 0x59, 0x95, 0x2a, 0xc1, 0xac, 0xc0, 0xa8, 0xc1, 0x1c, + 0x04, 0x66, 0x0b, 0xf9, 0x73, 0x71, 0x27, 0xe6, 0xe5, 0xe5, 0x97, 0x24, 0x96, 0x64, 0xe6, 0xe7, + 0x15, 0x4b, 0xb0, 0x2a, 0x30, 0x6b, 0x70, 0x1b, 0xe9, 0xea, 0xa1, 0xfb, 0x45, 0x0f, 0xe1, 0x62, + 0x3d, 0x47, 0x84, 0x7a, 0xd7, 0xbc, 0x92, 0xa2, 0xca, 0x20, 0x64, 0x13, 0xa4, 0xec, 0xb8, 0x04, + 0xd0, 0x15, 0x08, 0x09, 0x70, 0x31, 0x67, 0xa7, 0x56, 0x42, 0x3d, 0x07, 0x62, 0x0a, 0x89, 0x70, + 0xb1, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x42, 0x7c, 0x15, 0x04, 0xe1, 0x58, 0x31, 0x59, 0x30, 0x3a, + 0x79, 0x9d, 0x78, 0x28, 0xc7, 0x70, 0xe3, 0xa1, 0x1c, 0x43, 0xc3, 0x23, 0x39, 0xc6, 0x13, 0x8f, + 0xe4, 0x18, 0x2f, 0x3c, 0x92, 0x63, 0x7c, 0xf0, 0x48, 0x8e, 0x31, 0xca, 0x80, 0xf8, 0xd8, 0xb1, + 0x06, 0x93, 0x11, 0x0c, 0x49, 0x6c, 0xe0, 0x30, 0x37, 0x06, 0x04, 0x00, 0x00, 0xff, 0xff, 0x22, + 0x8a, 0x20, 0x4a, 0xda, 0x01, 0x00, 0x00, } diff --git a/api/types/descriptor.proto b/api/types/descriptor.proto index 5c00dca4f..6d90a1626 100644 --- a/api/types/descriptor.proto +++ b/api/types/descriptor.proto @@ -15,4 +15,5 @@ message Descriptor { string media_type = 1; string digest = 2 [(gogoproto.customtype) = "github.com/opencontainers/go-digest.Digest", (gogoproto.nullable) = false]; int64 size = 3; + map annotations = 5; } From 0b711d616ab13e9bad2801db7b53d5ffeb6934dc Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Wed, 6 Mar 2019 12:22:56 -0500 Subject: [PATCH 2/6] Copy annotations around where necessary Make sure that the newly added annotations are copied around appropriately. Signed-off-by: Stefan Berger --- container_checkpoint_opts.go | 9 +++++---- diff.go | 14 ++++++++------ image_store.go | 14 ++++++++------ services/diff/local.go | 14 ++++++++------ services/images/helpers.go | 14 ++++++++------ services/tasks/local.go | 14 ++++++++------ task.go | 1 + task_opts.go | 7 ++++--- 8 files changed, 50 insertions(+), 37 deletions(-) diff --git a/container_checkpoint_opts.go b/container_checkpoint_opts.go index 7d261421e..510863681 100644 --- a/container_checkpoint_opts.go +++ b/container_checkpoint_opts.go @@ -70,10 +70,11 @@ func WithCheckpointTask(ctx context.Context, client *Client, c *containers.Conta for _, d := range task.Descriptors { platformSpec := platforms.DefaultSpec() index.Manifests = append(index.Manifests, imagespec.Descriptor{ - MediaType: d.MediaType, - Size: d.Size_, - Digest: d.Digest, - Platform: &platformSpec, + MediaType: d.MediaType, + Size: d.Size_, + Digest: d.Digest, + Platform: &platformSpec, + Annotations: d.Annotations, }) } // save copts diff --git a/diff.go b/diff.go index 8d1219e35..4d890ce2b 100644 --- a/diff.go +++ b/diff.go @@ -80,17 +80,19 @@ func (r *diffRemote) Compare(ctx context.Context, a, b []mount.Mount, opts ...di func toDescriptor(d *types.Descriptor) ocispec.Descriptor { return ocispec.Descriptor{ - MediaType: d.MediaType, - Digest: d.Digest, - Size: d.Size_, + MediaType: d.MediaType, + Digest: d.Digest, + Size: d.Size_, + Annotations: d.Annotations, } } func fromDescriptor(d ocispec.Descriptor) *types.Descriptor { return &types.Descriptor{ - MediaType: d.MediaType, - Digest: d.Digest, - Size_: d.Size, + MediaType: d.MediaType, + Digest: d.Digest, + Size_: d.Size, + Annotations: d.Annotations, } } diff --git a/image_store.go b/image_store.go index 3676bdada..fd79e8929 100644 --- a/image_store.go +++ b/image_store.go @@ -137,16 +137,18 @@ func imagesFromProto(imagespb []imagesapi.Image) []images.Image { func descFromProto(desc *types.Descriptor) ocispec.Descriptor { return ocispec.Descriptor{ - MediaType: desc.MediaType, - Size: desc.Size_, - Digest: desc.Digest, + MediaType: desc.MediaType, + Size: desc.Size_, + Digest: desc.Digest, + Annotations: desc.Annotations, } } func descToProto(desc *ocispec.Descriptor) types.Descriptor { return types.Descriptor{ - MediaType: desc.MediaType, - Size_: desc.Size, - Digest: desc.Digest, + MediaType: desc.MediaType, + Size_: desc.Size, + Digest: desc.Digest, + Annotations: desc.Annotations, } } diff --git a/services/diff/local.go b/services/diff/local.go index 0cb6222c5..7cf6c768e 100644 --- a/services/diff/local.go +++ b/services/diff/local.go @@ -164,16 +164,18 @@ func toMounts(apim []*types.Mount) []mount.Mount { func toDescriptor(d *types.Descriptor) ocispec.Descriptor { return ocispec.Descriptor{ - MediaType: d.MediaType, - Digest: d.Digest, - Size: d.Size_, + MediaType: d.MediaType, + Digest: d.Digest, + Size: d.Size_, + Annotations: d.Annotations, } } func fromDescriptor(d ocispec.Descriptor) *types.Descriptor { return &types.Descriptor{ - MediaType: d.MediaType, - Digest: d.Digest, - Size_: d.Size, + MediaType: d.MediaType, + Digest: d.Digest, + Size_: d.Size, + Annotations: d.Annotations, } } diff --git a/services/images/helpers.go b/services/images/helpers.go index 8ad0d117e..2d4ec76dc 100644 --- a/services/images/helpers.go +++ b/services/images/helpers.go @@ -55,16 +55,18 @@ func imageFromProto(imagepb *imagesapi.Image) images.Image { func descFromProto(desc *types.Descriptor) ocispec.Descriptor { return ocispec.Descriptor{ - MediaType: desc.MediaType, - Size: desc.Size_, - Digest: desc.Digest, + MediaType: desc.MediaType, + Size: desc.Size_, + Digest: desc.Digest, + Annotations: desc.Annotations, } } func descToProto(desc *ocispec.Descriptor) types.Descriptor { return types.Descriptor{ - MediaType: desc.MediaType, - Size_: desc.Size, - Digest: desc.Digest, + MediaType: desc.MediaType, + Size_: desc.Size, + Digest: desc.Digest, + Annotations: desc.Annotations, } } diff --git a/services/tasks/local.go b/services/tasks/local.go index ee7dc1bc9..08628b1c0 100644 --- a/services/tasks/local.go +++ b/services/tasks/local.go @@ -143,9 +143,10 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. return nil, fmt.Errorf("unsupported checkpoint type %q", r.Checkpoint.MediaType) } reader, err := l.store.ReaderAt(ctx, ocispec.Descriptor{ - MediaType: r.Checkpoint.MediaType, - Digest: r.Checkpoint.Digest, - Size: r.Checkpoint.Size_, + MediaType: r.Checkpoint.MediaType, + Digest: r.Checkpoint.Digest, + Size: r.Checkpoint.Size_, + Annotations: r.Checkpoint.Annotations, }) if err != nil { return nil, err @@ -624,9 +625,10 @@ func (l *local) writeContent(ctx context.Context, mediaType, ref string, r io.Re return nil, err } return &types.Descriptor{ - MediaType: mediaType, - Digest: writer.Digest(), - Size_: size, + MediaType: mediaType, + Digest: writer.Digest(), + Size_: size, + Annotations: make(map[string]string), }, nil } diff --git a/task.go b/task.go index 9deb66c7e..679480e53 100644 --- a/task.go +++ b/task.go @@ -567,6 +567,7 @@ func (t *task) checkpointTask(ctx context.Context, index *v1.Index, request *tas OS: goruntime.GOOS, Architecture: goruntime.GOARCH, }, + Annotations: d.Annotations, }) } return nil diff --git a/task_opts.go b/task_opts.go index 714a152c5..8e005ac69 100644 --- a/task_opts.go +++ b/task_opts.go @@ -64,9 +64,10 @@ func WithTaskCheckpoint(im Image) NewTaskOpts { for _, m := range index.Manifests { if m.MediaType == images.MediaTypeContainerd1Checkpoint { info.Checkpoint = &types.Descriptor{ - MediaType: m.MediaType, - Size_: m.Size, - Digest: m.Digest, + MediaType: m.MediaType, + Size_: m.Size, + Digest: m.Digest, + Annotations: m.Annotations, } return nil } From 79248fea2b875f59c62ba5e085807d019bd81222 Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Wed, 6 Mar 2019 16:12:18 -0500 Subject: [PATCH 3/6] Add test for ocispec.Descriptor Annotations Make sure that Annotations we write into ocispec.Descriptors are written into the store and can be read back. Signed-off-by: Stefan Berger --- import_test.go | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/import_test.go b/import_test.go index c6cea6d54..2b9894d29 100644 --- a/import_test.go +++ b/import_test.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "math/rand" + "reflect" "runtime" "testing" @@ -94,11 +95,11 @@ func TestImport(t *testing.T) { c1, d2 := createConfig() - m1, d3 := createManifest(c1, [][]byte{b1}) + m1, d3, expManifest := createManifest(c1, [][]byte{b1}) provider := client.ContentStore() - checkManifest := func(ctx context.Context, t *testing.T, d ocispec.Descriptor) { + checkManifest := func(ctx context.Context, t *testing.T, d ocispec.Descriptor, expManifest *ocispec.Manifest) { m, err := images.Manifest(ctx, provider, d, nil) if err != nil { t.Fatalf("unable to read target blob: %+v", err) @@ -115,6 +116,15 @@ func TestImport(t *testing.T) { if m.Layers[0].Digest != d1 { t.Fatalf("unexpected layer hash %s, expected %s", m.Layers[0].Digest, d1) } + + if expManifest != nil { + if !reflect.DeepEqual(m.Layers, expManifest.Layers) { + t.Fatalf("DeepEqual on Layers failed: %v vs. %v", m.Layers, expManifest.Layers) + } + if !reflect.DeepEqual(m.Config, expManifest.Config) { + t.Fatalf("DeepEqual on Config failed: %v vs. %v", m.Config, expManifest.Config) + } + } } for _, tc := range []struct { @@ -154,7 +164,7 @@ func TestImport(t *testing.T) { } checkImages(t, imgs[0].Target.Digest, imgs, names...) - checkManifest(ctx, t, imgs[0].Target) + checkManifest(ctx, t, imgs[0].Target, nil) }, }, { @@ -181,7 +191,7 @@ func TestImport(t *testing.T) { } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target) + checkManifest(ctx, t, imgs[0].Target, expManifest) }, }, { @@ -202,7 +212,7 @@ func TestImport(t *testing.T) { } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target) + checkManifest(ctx, t, imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.AddRefPrefix("localhost:5000/myimage")), @@ -226,7 +236,7 @@ func TestImport(t *testing.T) { } checkImages(t, d3, imgs, names...) - checkManifest(ctx, t, imgs[0].Target) + checkManifest(ctx, t, imgs[0].Target, expManifest) }, Opts: []ImportOpt{ WithImageRefTranslator(archive.FilterRefPrefix("localhost:5000/myimage")), @@ -288,7 +298,7 @@ func createConfig() ([]byte, digest.Digest) { return b, digest.FromBytes(b) } -func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest) { +func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest, *ocispec.Manifest) { manifest := ocispec.Manifest{ Versioned: specs.Versioned{ SchemaVersion: 2, @@ -297,6 +307,9 @@ func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest) { MediaType: ocispec.MediaTypeImageConfig, Digest: digest.FromBytes(config), Size: int64(len(config)), + Annotations: map[string]string{ + "ocispec": "manifest.config.descriptor", + }, }, } for _, l := range layers { @@ -304,12 +317,15 @@ func createManifest(config []byte, layers [][]byte) ([]byte, digest.Digest) { MediaType: ocispec.MediaTypeImageLayer, Digest: digest.FromBytes(l), Size: int64(len(l)), + Annotations: map[string]string{ + "ocispec": "manifest.layers.descriptor", + }, }) } b, _ := json.Marshal(manifest) - return b, digest.FromBytes(b) + return b, digest.FromBytes(b), &manifest } func createIndex(manifest []byte, tags ...string) []byte { From 02cc1485dfae8e4798f153c89f521198354f7217 Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Thu, 7 Mar 2019 12:02:52 -0500 Subject: [PATCH 4/6] Prepare boltutil for reading and writing another map Refactor the code so that another function can also read and write maps into the bolt db. Signed-off-by: Stefan Berger --- metadata/boltutil/helpers.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/metadata/boltutil/helpers.go b/metadata/boltutil/helpers.go index 124018857..2b3c2bfb7 100644 --- a/metadata/boltutil/helpers.go +++ b/metadata/boltutil/helpers.go @@ -32,7 +32,11 @@ var ( // ReadLabels reads the labels key from the bucket // Uses the key "labels" func ReadLabels(bkt *bolt.Bucket) (map[string]string, error) { - lbkt := bkt.Bucket(bucketKeyLabels) + return readMap(bkt, bucketKeyLabels) +} + +func readMap(bkt *bolt.Bucket, bucketName []byte) (map[string]string, error) { + lbkt := bkt.Bucket(bucketName) if lbkt == nil { return nil, nil } @@ -53,9 +57,13 @@ func ReadLabels(bkt *bolt.Bucket) (map[string]string, error) { // bucket. Typically, this removes zero-value entries. // Uses the key "labels" func WriteLabels(bkt *bolt.Bucket, labels map[string]string) error { + return writeMap(bkt, bucketKeyLabels, labels) +} + +func writeMap(bkt *bolt.Bucket, bucketName []byte, labels map[string]string) error { // Remove existing labels to keep from merging - if lbkt := bkt.Bucket(bucketKeyLabels); lbkt != nil { - if err := bkt.DeleteBucket(bucketKeyLabels); err != nil { + if lbkt := bkt.Bucket(bucketName); lbkt != nil { + if err := bkt.DeleteBucket(bucketName); err != nil { return err } } @@ -64,7 +72,7 @@ func WriteLabels(bkt *bolt.Bucket, labels map[string]string) error { return nil } - lbkt, err := bkt.CreateBucket(bucketKeyLabels) + lbkt, err := bkt.CreateBucket(bucketName) if err != nil { return err } From 5124f9ee5472bfe1f61bbd37a1be879d1e5a14de Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Thu, 7 Mar 2019 13:53:48 -0500 Subject: [PATCH 5/6] Write the Annotations map into the bolt db Signed-off-by: Stefan Berger --- metadata/boltutil/helpers.go | 18 +++++++++++++++--- metadata/images.go | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/metadata/boltutil/helpers.go b/metadata/boltutil/helpers.go index 2b3c2bfb7..94af315f9 100644 --- a/metadata/boltutil/helpers.go +++ b/metadata/boltutil/helpers.go @@ -24,9 +24,10 @@ import ( ) var ( - bucketKeyLabels = []byte("labels") - bucketKeyCreatedAt = []byte("createdat") - bucketKeyUpdatedAt = []byte("updatedat") + bucketKeyAnnotations = []byte("annotations") + bucketKeyLabels = []byte("labels") + bucketKeyCreatedAt = []byte("createdat") + bucketKeyUpdatedAt = []byte("updatedat") ) // ReadLabels reads the labels key from the bucket @@ -35,6 +36,12 @@ func ReadLabels(bkt *bolt.Bucket) (map[string]string, error) { return readMap(bkt, bucketKeyLabels) } +// ReadAnnotations reads the OCI Descriptor Annotations key from the bucket +// Uses the key "annotations" +func ReadAnnotations(bkt *bolt.Bucket) (map[string]string, error) { + return readMap(bkt, bucketKeyAnnotations) +} + func readMap(bkt *bolt.Bucket, bucketName []byte) (map[string]string, error) { lbkt := bkt.Bucket(bucketName) if lbkt == nil { @@ -60,6 +67,11 @@ func WriteLabels(bkt *bolt.Bucket, labels map[string]string) error { return writeMap(bkt, bucketKeyLabels, labels) } +// WriteAnnotations writes the OCI Descriptor Annotations +func WriteAnnotations(bkt *bolt.Bucket, labels map[string]string) error { + return writeMap(bkt, bucketKeyAnnotations, labels) +} + func writeMap(bkt *bolt.Bucket, bucketName []byte, labels map[string]string) error { // Remove existing labels to keep from merging if lbkt := bkt.Bucket(bucketName); lbkt != nil { diff --git a/metadata/images.go b/metadata/images.go index 762f6fb1a..1dda753db 100644 --- a/metadata/images.go +++ b/metadata/images.go @@ -192,6 +192,14 @@ func (s *imageStore) Update(ctx context.Context, image images.Image, fieldpaths key := strings.TrimPrefix(path, "labels.") updated.Labels[key] = image.Labels[key] continue + } else if strings.HasPrefix(path, "annotations.") { + if updated.Target.Annotations == nil { + updated.Target.Annotations = map[string]string{} + } + + key := strings.TrimPrefix(path, "annotations.") + updated.Target.Annotations[key] = image.Target.Annotations[key] + continue } switch path { @@ -204,6 +212,8 @@ func (s *imageStore) Update(ctx context.Context, image images.Image, fieldpaths // make sense to modify the size or digest without touching the // mediatype, as well, for example. updated.Target = image.Target + case "annotations": + updated.Target.Annotations = image.Target.Annotations default: return errors.Wrapf(errdefs.ErrInvalidArgument, "cannot update %q field on image %q", path, image.Name) } @@ -298,6 +308,11 @@ func readImage(image *images.Image, bkt *bolt.Bucket) error { } image.Labels = labels + image.Target.Annotations, err = boltutil.ReadAnnotations(bkt) + if err != nil { + return err + } + tbkt := bkt.Bucket(bucketKeyTarget) if tbkt == nil { return errors.New("unable to read target bucket") @@ -331,6 +346,10 @@ func writeImage(bkt *bolt.Bucket, image *images.Image) error { return errors.Wrapf(err, "writing labels for image %v", image.Name) } + if err := boltutil.WriteAnnotations(bkt, image.Target.Annotations); err != nil { + return errors.Wrapf(err, "writing Annotations for image %v", image.Name) + } + // write the target bucket tbkt, err := bkt.CreateBucketIfNotExists(bucketKeyTarget) if err != nil { From 09cf2a629b2a10ba6c0352d06eb6838544bbdcb9 Mon Sep 17 00:00:00 2001 From: Stefan Berger Date: Thu, 7 Mar 2019 13:54:41 -0500 Subject: [PATCH 6/6] Extend metadata images test with fieldpaths for Annotations Signed-off-by: Stefan Berger --- metadata/images_test.go | 116 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/metadata/images_test.go b/metadata/images_test.go index 1bd92cf37..7a855a8e2 100644 --- a/metadata/images_test.go +++ b/metadata/images_test.go @@ -49,6 +49,9 @@ func TestImagesList(t *testing.T) { Size: 10, MediaType: "application/vnd.containerd.test", Digest: digest.FromString(id), + Annotations: map[string]string{ + "foo": "bar", + }, }, } @@ -168,6 +171,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, expected: images.Image{ @@ -178,6 +184,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, }, @@ -203,6 +212,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, expected: images.Image{ @@ -213,6 +225,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, }, @@ -227,6 +242,10 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 20, // ignored MediaType: "application/vnd.oci.blab+ignored", // make sure other stuff is ignored + Annotations: map[string]string{ + "not": "bar", + "new": "boo", + }, }, }, fieldpaths: []string{"labels"}, @@ -238,6 +257,41 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + "baz": "boo", + }, + }, + }, + }, + { + name: "ReplaceLabelsAnnotationsFieldPath", + original: imageBase(), + input: images.Image{ + Labels: map[string]string{ + "for": "bar", + "boo": "boo", + }, + Target: ocispec.Descriptor{ + Size: 20, // ignored + MediaType: "application/vnd.oci.blab+ignored", // make sure other stuff is ignored + Annotations: map[string]string{ + "foo": "boo", + }, + }, + }, + fieldpaths: []string{"annotations", "labels"}, + expected: images.Image{ + Labels: map[string]string{ + "for": "bar", + "boo": "boo", + }, + Target: ocispec.Descriptor{ + Size: 10, + MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "boo", + }, }, }, }, @@ -252,6 +306,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 20, // ignored MediaType: "application/vnd.oci.blab+ignored", // make sure other stuff is ignored + Annotations: map[string]string{ + "foo": "bar", + }, }, }, fieldpaths: []string{"labels.foo"}, @@ -263,6 +320,43 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + "baz": "boo", + }, + }, + }, + }, + { + name: "ReplaceAnnotation", + original: imageBase(), + input: images.Image{ + Labels: map[string]string{ + "foo": "baz", + "baz": "bunk", + }, + Target: ocispec.Descriptor{ + Size: 20, // ignored + MediaType: "application/vnd.oci.blab+ignored", // make sure other stuff is ignored + Annotations: map[string]string{ + "foo": "baz", + "baz": "bunk", + }, + }, + }, + fieldpaths: []string{"annotations.foo"}, + expected: images.Image{ + Labels: map[string]string{ + "foo": "bar", + "baz": "boo", + }, + Target: ocispec.Descriptor{ + Size: 10, + MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "baz", + "baz": "boo", + }, }, }, }, @@ -273,6 +367,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab+replaced", + Annotations: map[string]string{ + "fox": "dog", + }, }, }, fieldpaths: []string{"target"}, @@ -284,6 +381,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab+replaced", + Annotations: map[string]string{ + "fox": "dog", + }, }, }, }, @@ -298,6 +398,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 0, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, cause: errdefs.ErrInvalidArgument, @@ -311,6 +414,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { }, Target: ocispec.Descriptor{ MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, createerr: errdefs.ErrInvalidArgument, @@ -352,6 +458,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, input: images.Image{ @@ -363,6 +472,9 @@ func TestImagesCreateUpdateDelete(t *testing.T) { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + }, }, }, cause: errdefs.ErrNotFound, @@ -440,6 +552,10 @@ func imageBase() images.Image { Target: ocispec.Descriptor{ Size: 10, MediaType: "application/vnd.oci.blab", + Annotations: map[string]string{ + "foo": "bar", + "baz": "boo", + }, }, } }