From 081601f521e252afb7cdf79e5fa9098e0a7b2129 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 11 Jan 2023 08:12:36 -0800 Subject: [PATCH] Update imagestore interface to support multiple references Signed-off-by: Derek McGowan --- api/next.pb.txt | 68 ++++--- api/types/transfer/imagestore.pb.go | 243 +++++++++++++++------- api/types/transfer/imagestore.proto | 40 +++- cmd/ctr/commands/images/import.go | 11 +- integration/client/import_test.go | 249 ++++++++++++++++++++++- pkg/transfer/image/imagestore.go | 277 +++++++++++++++++++------- pkg/transfer/image/imagestore_test.go | 275 +++++++++++++++++++++++++ pkg/transfer/local/import.go | 25 ++- pkg/transfer/local/pull.go | 13 +- pkg/transfer/transfer.go | 7 +- 10 files changed, 996 insertions(+), 212 deletions(-) create mode 100644 pkg/transfer/image/imagestore_test.go diff --git a/api/next.pb.txt b/api/next.pb.txt index 534a6cbed..10952d41b 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -6757,32 +6757,12 @@ file { json_name: "manifestLimit" } field { - name: "prefix" + name: "extra_references" number: 6 - label: LABEL_OPTIONAL - type: TYPE_STRING - json_name: "prefix" - } - field { - name: "check_prefix" - number: 7 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "checkPrefix" - } - field { - name: "digest_refs" - number: 8 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "digestRefs" - } - field { - name: "always_digest" - number: 9 - label: LABEL_OPTIONAL - type: TYPE_BOOL - json_name: "alwaysDigest" + label: LABEL_REPEATED + type: TYPE_MESSAGE + type_name: ".containerd.types.transfer.ImageReference" + json_name: "extraReferences" } field { name: "unpacks" @@ -6831,6 +6811,44 @@ file { json_name: "snapshotter" } } + message_type { + name: "ImageReference" + field { + name: "name" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "name" + } + field { + name: "is_prefix" + number: 2 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "isPrefix" + } + field { + name: "allow_overwrite" + number: 3 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "allowOverwrite" + } + field { + name: "add_digest" + number: 4 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "addDigest" + } + field { + name: "skip_named_digest" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "skipNamedDigest" + } + } options { go_package: "github.com/containerd/containerd/api/types/transfer" } diff --git a/api/types/transfer/imagestore.pb.go b/api/types/transfer/imagestore.pb.go index 77826e4e2..b3c983041 100644 --- a/api/types/transfer/imagestore.pb.go +++ b/api/types/transfer/imagestore.pb.go @@ -46,15 +46,9 @@ type ImageStore struct { Platforms []*types.Platform `protobuf:"bytes,3,rep,name=platforms,proto3" json:"platforms,omitempty"` AllMetadata bool `protobuf:"varint,4,opt,name=all_metadata,json=allMetadata,proto3" json:"all_metadata,omitempty"` ManifestLimit uint32 `protobuf:"varint,5,opt,name=manifest_limit,json=manifestLimit,proto3" json:"manifest_limit,omitempty"` - // prefix is the intended image name prefix for imported images - Prefix string `protobuf:"bytes,6,opt,name=prefix,proto3" json:"prefix,omitempty"` - // check_prefix only stores images with the prefix - CheckPrefix bool `protobuf:"varint,7,opt,name=check_prefix,json=checkPrefix,proto3" json:"check_prefix,omitempty"` - // digest_refs adds digest references for images using prefix - DigestRefs bool `protobuf:"varint,8,opt,name=digest_refs,json=digestRefs,proto3" json:"digest_refs,omitempty"` - // always_digest includes a digest image even when a non-digest image is stored - AlwaysDigest bool `protobuf:"varint,9,opt,name=always_digest,json=alwaysDigest,proto3" json:"always_digest,omitempty"` - Unpacks []*UnpackConfiguration `protobuf:"bytes,10,rep,name=unpacks,proto3" json:"unpacks,omitempty"` + // extra_references are used to set image names on imports of sub-images from the index + ExtraReferences []*ImageReference `protobuf:"bytes,6,rep,name=extra_references,json=extraReferences,proto3" json:"extra_references,omitempty"` + Unpacks []*UnpackConfiguration `protobuf:"bytes,10,rep,name=unpacks,proto3" json:"unpacks,omitempty"` } func (x *ImageStore) Reset() { @@ -124,32 +118,11 @@ func (x *ImageStore) GetManifestLimit() uint32 { return 0 } -func (x *ImageStore) GetPrefix() string { +func (x *ImageStore) GetExtraReferences() []*ImageReference { if x != nil { - return x.Prefix + return x.ExtraReferences } - return "" -} - -func (x *ImageStore) GetCheckPrefix() bool { - if x != nil { - return x.CheckPrefix - } - return false -} - -func (x *ImageStore) GetDigestRefs() bool { - if x != nil { - return x.DigestRefs - } - return false -} - -func (x *ImageStore) GetAlwaysDigest() bool { - if x != nil { - return x.AlwaysDigest - } - return false + return nil } func (x *ImageStore) GetUnpacks() []*UnpackConfiguration { @@ -217,6 +190,103 @@ func (x *UnpackConfiguration) GetSnapshotter() string { return "" } +// ImageReference is used to create or find a reference for an image +type ImageReference struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // is_prefix determines whether the Name should be considered + // a prefix (without tag or digest). + // For lookup, this may allow matching multiple tags. + // For store, this must have a tag or digest added. + IsPrefix bool `protobuf:"varint,2,opt,name=is_prefix,json=isPrefix,proto3" json:"is_prefix,omitempty"` + // allow_overwrite allows overwriting or ignoring the name if + // another reference is provided (such as through an annotation). + // Only used if IsPrefix is true. + AllowOverwrite bool `protobuf:"varint,3,opt,name=allow_overwrite,json=allowOverwrite,proto3" json:"allow_overwrite,omitempty"` + // add_digest adds the manifest digest to the reference. + // For lookup, this allows matching tags with any digest. + // For store, this allows adding the digest to the name. + // Only used if IsPrefix is true. + AddDigest bool `protobuf:"varint,4,opt,name=add_digest,json=addDigest,proto3" json:"add_digest,omitempty"` + // skip_named_digest only considers digest references which do not + // have a non-digested named reference. + // For lookup, this will deduplicate digest references when there is a named match. + // For store, this only adds this digest reference when there is no matching full + // name reference from the prefix. + // Only used if IsPrefix is true. + SkipNamedDigest bool `protobuf:"varint,5,opt,name=skip_named_digest,json=skipNamedDigest,proto3" json:"skip_named_digest,omitempty"` +} + +func (x *ImageReference) Reset() { + *x = ImageReference{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ImageReference) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImageReference) ProtoMessage() {} + +func (x *ImageReference) ProtoReflect() protoreflect.Message { + mi := &file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImageReference.ProtoReflect.Descriptor instead. +func (*ImageReference) Descriptor() ([]byte, []int) { + return file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_rawDescGZIP(), []int{2} +} + +func (x *ImageReference) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ImageReference) GetIsPrefix() bool { + if x != nil { + return x.IsPrefix + } + return false +} + +func (x *ImageReference) GetAllowOverwrite() bool { + if x != nil { + return x.AllowOverwrite + } + return false +} + +func (x *ImageReference) GetAddDigest() bool { + if x != nil { + return x.AddDigest + } + return false +} + +func (x *ImageReference) GetSkipNamedDigest() bool { + if x != nil { + return x.SkipNamedDigest + } + return false +} + var File_github_com_containerd_containerd_api_types_transfer_imagestore_proto protoreflect.FileDescriptor var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_rawDesc = []byte{ @@ -229,7 +299,7 @@ var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_ra 0x72, 0x1a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x70, 0x6c, - 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xf5, 0x03, 0x0a, + 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xca, 0x03, 0x0a, 0x0a, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x49, 0x0a, 0x06, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, @@ -244,35 +314,44 @@ var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_ra 0x64, 0x61, 0x74, 0x61, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x61, 0x6c, 0x6c, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, - 0x0d, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, - 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x68, - 0x65, 0x63, 0x6b, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1f, 0x0a, 0x0b, 0x64, 0x69, 0x67, - 0x65, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x66, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, - 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x52, 0x65, 0x66, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, - 0x77, 0x61, 0x79, 0x73, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0c, 0x61, 0x6c, 0x77, 0x61, 0x79, 0x73, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, - 0x48, 0x0a, 0x07, 0x75, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x73, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2e, 0x55, 0x6e, 0x70, + 0x0d, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x54, + 0x0a, 0x10, 0x65, 0x78, 0x74, 0x72, 0x61, 0x5f, 0x72, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x52, 0x0f, 0x65, 0x78, 0x74, 0x72, 0x61, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, + 0x6e, 0x63, 0x65, 0x73, 0x12, 0x48, 0x0a, 0x07, 0x75, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x73, 0x18, + 0x0a, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, + 0x72, 0x2e, 0x55, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x75, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x73, 0x1a, 0x39, + 0x0a, 0x0b, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x6f, 0x0a, 0x13, 0x55, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x07, 0x75, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, - 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x3a, 0x02, 0x38, 0x01, 0x22, 0x6f, 0x0a, 0x13, 0x55, 0x6e, 0x70, 0x61, 0x63, 0x6b, 0x43, 0x6f, - 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x0a, 0x08, 0x70, - 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, - 0x6f, 0x72, 0x6d, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, - 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, - 0x6f, 0x74, 0x74, 0x65, 0x72, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x36, 0x0a, 0x08, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, + 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x20, 0x0a, 0x0b, 0x73, 0x6e, 0x61, 0x70, + 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, + 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x74, 0x65, 0x72, 0x22, 0xb5, 0x01, 0x0a, 0x0e, 0x49, + 0x6d, 0x61, 0x67, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x73, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x27, + 0x0a, 0x0f, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x6f, 0x76, 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x4f, 0x76, + 0x65, 0x72, 0x77, 0x72, 0x69, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x64, 0x64, 0x5f, 0x64, + 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x61, 0x64, 0x64, + 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x64, 0x5f, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0f, 0x73, 0x6b, 0x69, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x64, 0x44, 0x69, 0x67, 0x65, + 0x73, 0x74, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, + 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, } var ( @@ -287,23 +366,25 @@ func file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_r return file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_rawDescData } -var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_msgTypes = make([]protoimpl.MessageInfo, 4) var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_goTypes = []interface{}{ (*ImageStore)(nil), // 0: containerd.types.transfer.ImageStore (*UnpackConfiguration)(nil), // 1: containerd.types.transfer.UnpackConfiguration - nil, // 2: containerd.types.transfer.ImageStore.LabelsEntry - (*types.Platform)(nil), // 3: containerd.types.Platform + (*ImageReference)(nil), // 2: containerd.types.transfer.ImageReference + nil, // 3: containerd.types.transfer.ImageStore.LabelsEntry + (*types.Platform)(nil), // 4: containerd.types.Platform } var file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_depIdxs = []int32{ - 2, // 0: containerd.types.transfer.ImageStore.labels:type_name -> containerd.types.transfer.ImageStore.LabelsEntry - 3, // 1: containerd.types.transfer.ImageStore.platforms:type_name -> containerd.types.Platform - 1, // 2: containerd.types.transfer.ImageStore.unpacks:type_name -> containerd.types.transfer.UnpackConfiguration - 3, // 3: containerd.types.transfer.UnpackConfiguration.platform:type_name -> containerd.types.Platform - 4, // [4:4] is the sub-list for method output_type - 4, // [4:4] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 3, // 0: containerd.types.transfer.ImageStore.labels:type_name -> containerd.types.transfer.ImageStore.LabelsEntry + 4, // 1: containerd.types.transfer.ImageStore.platforms:type_name -> containerd.types.Platform + 2, // 2: containerd.types.transfer.ImageStore.extra_references:type_name -> containerd.types.transfer.ImageReference + 1, // 3: containerd.types.transfer.ImageStore.unpacks:type_name -> containerd.types.transfer.UnpackConfiguration + 4, // 4: containerd.types.transfer.UnpackConfiguration.platform:type_name -> containerd.types.Platform + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_init() } @@ -336,6 +417,18 @@ func file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_i return nil } } + file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ImageReference); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -343,7 +436,7 @@ func file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_i GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_github_com_containerd_containerd_api_types_transfer_imagestore_proto_rawDesc, NumEnums: 0, - NumMessages: 3, + NumMessages: 4, NumExtensions: 0, NumServices: 0, }, diff --git a/api/types/transfer/imagestore.proto b/api/types/transfer/imagestore.proto index 971410b62..57ac2ebde 100644 --- a/api/types/transfer/imagestore.proto +++ b/api/types/transfer/imagestore.proto @@ -34,14 +34,8 @@ message ImageStore { // Import naming - // prefix is the intended image name prefix for imported images - string prefix = 6; - // check_prefix only stores images with the prefix - bool check_prefix = 7; - // digest_refs adds digest references for images using prefix - bool digest_refs = 8; - // always_digest includes a digest image even when a non-digest image is stored - bool always_digest = 9; + // extra_references are used to set image names on imports of sub-images from the index + repeated ImageReference extra_references = 6; // Unpack Configuration, multiple allowed @@ -56,3 +50,33 @@ message UnpackConfiguration { // snapshotter to unpack to, if not provided default for platform shoudl be used string snapshotter = 2; } + +// ImageReference is used to create or find a reference for an image +message ImageReference { + string name = 1; + + // is_prefix determines whether the Name should be considered + // a prefix (without tag or digest). + // For lookup, this may allow matching multiple tags. + // For store, this must have a tag or digest added. + bool is_prefix = 2; + + // allow_overwrite allows overwriting or ignoring the name if + // another reference is provided (such as through an annotation). + // Only used if IsPrefix is true. + bool allow_overwrite = 3; + + // add_digest adds the manifest digest to the reference. + // For lookup, this allows matching tags with any digest. + // For store, this allows adding the digest to the name. + // Only used if IsPrefix is true. + bool add_digest = 4; + + // skip_named_digest only considers digest references which do not + // have a non-digested named reference. + // For lookup, this will deduplicate digest references when there is a named match. + // For store, this only adds this digest reference when there is no matching full + // name reference from the prefix. + // Only used if IsPrefix is true. + bool skip_named_digest = 5; +} diff --git a/cmd/ctr/commands/images/import.go b/cmd/ctr/commands/images/import.go index cde0bd1ce..aa3ecb364 100644 --- a/cmd/ctr/commands/images/import.go +++ b/cmd/ctr/commands/images/import.go @@ -114,16 +114,17 @@ If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadb if !context.BoolT("local") { var opts []image.StoreOpt prefix := context.String("base-name") + var overwrite bool if prefix == "" { prefix = fmt.Sprintf("import-%s", time.Now().Format("2006-01-02")) - opts = append(opts, image.WithNamePrefix(prefix, false)) - } else { - // When provided, filter out references which do not match - opts = append(opts, image.WithNamePrefix(prefix, true)) + // Allow overwriting auto-generated prefix with named annotation + overwrite = true } if context.Bool("digests") { - opts = append(opts, image.WithDigestRefs(!context.Bool("skip-digest-for-named"))) + opts = append(opts, image.WithDigestRef(prefix, overwrite, !context.Bool("skip-digest-for-named"))) + } else { + opts = append(opts, image.WithNamedPrefix(prefix, overwrite)) } // TODO: Add platform options diff --git a/integration/client/import_test.go b/integration/client/import_test.go index 7de3ae0b4..f2ba3db35 100644 --- a/integration/client/import_test.go +++ b/integration/client/import_test.go @@ -20,12 +20,15 @@ import ( "bytes" "context" "encoding/json" + "fmt" + "hash/fnv" "io" - "os" - "math/rand" + "os" "reflect" "runtime" + "strings" + "sync" "testing" "time" @@ -36,6 +39,9 @@ import ( "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/leases" "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/transfer" + tarchive "github.com/containerd/containerd/pkg/transfer/archive" + "github.com/containerd/containerd/pkg/transfer/image" "github.com/containerd/containerd/platforms" digest "github.com/opencontainers/go-digest" @@ -165,8 +171,8 @@ func TestImport(t *testing.T) { empty := []byte("{}") version := []byte("1.0") - c1, d2 := createConfig(runtime.GOOS, runtime.GOARCH) - badConfig, _ := createConfig("foo", "lish") + c1, d2 := createConfig(runtime.GOOS, runtime.GOARCH, "test") + badConfig, _ := createConfig("foo", "lish", "test") m1, d3, expManifest := createManifest(c1, [][]byte{b1}) @@ -381,11 +387,11 @@ func createContent(size int64, seed int64) ([]byte, digest.Digest) { return b, digest.FromBytes(b) } -func createConfig(osName, archName string) ([]byte, digest.Digest) { +func createConfig(osName, archName, author string) ([]byte, digest.Digest) { image := ocispec.Image{ OS: osName, Architecture: archName, - Author: "test", + Author: author, } b, _ := json.Marshal(image) @@ -450,3 +456,234 @@ func createIndex(manifest []byte, tags ...string) []byte { return b } + +func TestTransferImport(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + + client, err := newClient(t, address) + if err != nil { + t.Fatal(err) + } + defer client.Close() + + for _, testCase := range []struct { + // Name is the name of the test + Name string + + // Images is the names of the images to create + // [0]: Index name or "" + // [1:]: Additional images and manifest to import + // Images ending with @ will have digest appended and use the digest of the previously imported image + // A space can be used to seperate a repo name and tag, only the tag will be set in the imported image + Images []string + Opts []image.StoreOpt + }{ + { + Name: "Basic", + Images: []string{"", "registry.test/basic:latest"}, + Opts: []image.StoreOpt{image.WithNamedPrefix("unused", true)}, + }, + { + Name: "IndexRef", + Images: []string{"registry.test/index-ref:latest", ""}, + }, + { + Name: "AllRefs", + Images: []string{"registry.test/all-refs:index", "registry.test/all-refs:1"}, + Opts: []image.StoreOpt{image.WithNamedPrefix("registry.test/all-refs", false)}, + }, + { + Name: "DigestRefs", + Images: []string{"registry.test/all-refs:index", "registry.test/all-refs:1", "registry.test/all-refs@"}, + Opts: []image.StoreOpt{image.WithDigestRef("registry.test/all-refs", false, false)}, + }, + { + Name: "DigestRefsSkipNamed", + Images: []string{"registry.test/all-refs:index", "registry.test/all-refs:1"}, + Opts: []image.StoreOpt{image.WithDigestRef("registry.test/all-refs", false, true)}, + }, + { + Name: "DigestOnly", + Images: []string{"", "", "imported-image@"}, + Opts: []image.StoreOpt{image.WithDigestRef("imported-image", false, true)}, + }, + { + Name: "OverwriteDigestRefs", + Images: []string{"registry.test/all-refs:index", "registry.test/all-refs:1", "someimportname@"}, + Opts: []image.StoreOpt{image.WithDigestRef("someimportname", true, false)}, + }, + { + Name: "TagOnlyRef", + Images: []string{"", "registry.test/myimage thebest"}, + Opts: []image.StoreOpt{image.WithNamedPrefix("registry.test/myimage", false)}, + }, + { + Name: "TagOnlyOverwriteDigestRefs", + Images: []string{"registry.test/all-refs:index", "registry.test/basename latest", "registry.test/basename@"}, + Opts: []image.StoreOpt{image.WithDigestRef("registry.test/basename", true, false)}, + }, + } { + testCase := testCase + t.Run(testCase.Name, func(t *testing.T) { + tc := tartest.TarContext{} + files := []tartest.WriterToTar{ + tc.Dir("blobs", 0755), + tc.Dir("blobs/sha256", 0755), + } + + descs, tws := createImages(tc, testCase.Images...) + files = append(files, tws...) + + files = append(files, tc.File("oci-layout", []byte(`{"imageLayoutVersion":"1.0.0"}`), 0644)) + + r := tartest.TarFromWriterTo(tartest.TarAll(files...)) + + var idxName string + if len(testCase.Images) > 0 { + idxName = testCase.Images[0] + } + + is := image.NewStore(idxName, testCase.Opts...) + + iis := tarchive.NewImageImportStream(r, "") + + progressTracker := &imagesProgress{} + + err := client.Transfer(ctx, iis, is, transfer.WithProgress(progressTracker.Progress)) + closeErr := r.Close() + if err != nil { + t.Fatal(err) + } + if closeErr != nil { + t.Fatal(closeErr) + } + + imgs := progressTracker.getImages() + if len(descs) != len(imgs) { + t.Fatalf("unexpected number of images saved:\n\t(%d) %v\nexpected image map:\n\t(%d) %v", len(imgs), imgs, len(descs), descs) + } + store := client.ImageService() + for _, image := range imgs { + desc, ok := descs[image] + if !ok { + t.Fatalf("saved image %q not found in expected list\nimages saved:\n\t(%d) %v\nexpected image map:\n\t(%d) %v", image, len(progressTracker.images), progressTracker.images, len(descs), descs) + } + img, err := store.Get(ctx, image) + if err != nil { + t.Fatalf("error getting image %s: %v", image, err) + } + if img.Target.Digest != desc.Digest { + t.Fatalf("digests don't match for %s: got %s, expected %s", image, img.Target.Digest, desc.Digest) + } + if img.Target.MediaType != desc.MediaType { + t.Fatalf("media type don't match for %s: got %s, expected %s", image, img.Target.MediaType, desc.MediaType) + } + if img.Target.Size != desc.Size { + t.Fatalf("size don't match for %s: got %d, expected %d", image, img.Target.Size, desc.Size) + } + } + }) + } +} + +type imagesProgress struct { + sync.Mutex + images []string +} + +func (ip *imagesProgress) Progress(p transfer.Progress) { + ip.Lock() + if p.Event == "saved" { + ip.images = append(ip.images, p.Name) + } + ip.Unlock() +} + +func (ip *imagesProgress) getImages() []string { + ip.Lock() + imgs := ip.images + ip.Unlock() + return imgs + +} + +func createImages(tc tartest.TarContext, imageNames ...string) (descs map[string]ocispec.Descriptor, tw []tartest.WriterToTar) { + descs = map[string]ocispec.Descriptor{} + idx := ocispec.Index{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + } + + if len(imageNames) > 1 { + var lastManifest ocispec.Descriptor + + for _, image := range imageNames[1:] { + if image != "" && image[len(image)-1] == '@' { + image = image[:len(image)-1] + descs[fmt.Sprintf("%s@%s", image, lastManifest.Digest)] = lastManifest + continue + } + seed := hash64(image) + bb, b := createContent(128, seed) + tw = append(tw, tc.File("blobs/sha256/"+b.Encoded(), bb, 0644)) + + cb, c := createConfig("linux", "amd64", image) + tw = append(tw, tc.File("blobs/sha256/"+c.Encoded(), cb, 0644)) + + mb, m, _ := createManifest(cb, [][]byte{bb}) + tw = append(tw, tc.File("blobs/sha256/"+m.Encoded(), mb, 0644)) + + annotations := map[string]string{} + if image != "" { + if parts := strings.SplitN(image, " ", 2); len(parts) == 2 { + annotations[ocispec.AnnotationRefName] = parts[1] + image = strings.Join(parts, ":") + } else { + annotations[images.AnnotationImageName] = image + } + } + + md := ocispec.Descriptor{ + Digest: m, + Size: int64(len(mb)), + MediaType: ocispec.MediaTypeImageManifest, + Annotations: annotations, + } + + // If image is empty, but has base and digest, still use digest + // If image is not a full reference, then add base if provided? + if image != "" { + descs[image] = md + } + + idx.Manifests = append(idx.Manifests, md) + lastManifest = md + } + } + + ib, _ := json.Marshal(idx) + id := ocispec.Descriptor{ + Digest: digest.FromBytes(ib), + Size: int64(len(ib)), + MediaType: ocispec.MediaTypeImageIndex, + } + tw = append(tw, tc.File("index.json", ib, 0644)) + + var idxName string + if len(imageNames) > 0 { + idxName = imageNames[0] + } + if idxName != "" { + descs[idxName] = id + } + + return +} + +func hash64(s string) int64 { + h := fnv.New64a() + h.Write([]byte(s)) + return int64(h.Sum64()) +} diff --git a/pkg/transfer/image/imagestore.go b/pkg/transfer/image/imagestore.go index 953803b67..ff5299d56 100644 --- a/pkg/transfer/image/imagestore.go +++ b/pkg/transfer/image/imagestore.go @@ -48,15 +48,42 @@ type Store struct { labelMap func(ocispec.Descriptor) []string manifestLimit int - //import image options - namePrefix string - checkPrefix bool - digestRefs bool - alwaysDigest bool + // extraReferences are used to store or lookup multiple references + extraReferences []Reference unpacks []UnpackConfiguration } +// Reference is used to create or find a reference for an image +type Reference struct { + Name string + + // IsPrefix determines whether the Name should be considered + // a prefix (without tag or digest). + // For lookup, this may allow matching multiple tags. + // For store, this must have a tag or digest added. + IsPrefix bool + + // AllowOverwrite allows overwriting or ignoring the name if + // another reference is provided (such as through an annotation). + // Only used if IsPrefix is true. + AllowOverwrite bool + + // AddDigest adds the manifest digest to the reference. + // For lookup, this allows matching tags with any digest. + // For store, this allows adding the digest to the name. + // Only used if IsPrefix is true. + AddDigest bool + + // SkipNamedDigest only considers digest references which do not + // have a non-digested named reference. + // For lookup, this will deduplicate digest references when there is a named match. + // For store, this only adds this digest reference when there is no matching full + // name reference from the prefix. + // Only used if IsPrefix is true. + SkipNamedDigest bool +} + // UnpackConfiguration specifies the platform and snapshotter to use for resolving // the unpack Platform, if snapshotter is not specified the platform default will // be used. @@ -93,22 +120,51 @@ func WithAllMetadata(s *Store) { s.allMetadata = true } -// WithNamePrefix sets the name prefix for imported images, if -// check is enabled, then only images with the prefix are stored. -func WithNamePrefix(prefix string, check bool) StoreOpt { +// WithNamedPrefix uses a named prefix to references images which only have a tag name +// reference in the annotation or check full references annotations against. Images +// with no reference resolved from matching annotations will not be stored. +// - name: image name prefix to append a tag to or check full name references with +// - allowOverwrite: allows the tag to be overwritten by full name reference inside +// the image which does not have name as the prefix +func WithNamedPrefix(name string, allowOverwrite bool) StoreOpt { + ref := Reference{ + Name: name, + IsPrefix: true, + AllowOverwrite: allowOverwrite, + } return func(s *Store) { - s.namePrefix = prefix - s.checkPrefix = check + s.extraReferences = append(s.extraReferences, ref) } } -// WithDigestRefs sets digest refs for imported images, if -// always is enabled, then digest refs are added even if a -// non-digest image name is added for the same image. -func WithDigestRefs(always bool) StoreOpt { +// WithNamedPrefix uses a named prefix to references images which only have a tag name +// reference in the annotation or check full references annotations against and +// additionally may add a digest reference. Images with no references resolved +// from matching annotations may be stored by digest. +// - name: image name prefix to append a tag to or check full name references with +// - allowOverwrite: allows the tag to be overwritten by full name reference inside +// the image which does not have name as the prefix +// - skipNamed: is set if no digest reference should be created if a named reference +// is successfully resolved from the annotations. +func WithDigestRef(name string, allowOverwrite bool, skipNamed bool) StoreOpt { + ref := Reference{ + Name: name, + IsPrefix: true, + AllowOverwrite: allowOverwrite, + AddDigest: true, + SkipNamedDigest: skipNamed, + } return func(s *Store) { - s.digestRefs = true - s.alwaysDigest = always + s.extraReferences = append(s.extraReferences, ref) + } +} + +func WithExtraReference(name string) StoreOpt { + ref := Reference{ + Name: name, + } + return func(s *Store) { + s.extraReferences = append(s.extraReferences, ref) } } @@ -163,64 +219,114 @@ func (is *Store) ImageFilter(h images.HandlerFunc, cs content.Store) images.Hand return h } -func (is *Store) Store(ctx context.Context, desc ocispec.Descriptor, store images.Store) (images.Image, error) { - img := images.Image{ - Name: is.imageName, - Target: desc, - Labels: is.imageLabels, - } +func (is *Store) Store(ctx context.Context, desc ocispec.Descriptor, store images.Store) ([]images.Image, error) { + var imgs []images.Image - // Handle imported image names - if refType, ok := desc.Annotations["io.containerd.import.ref-type"]; ok { - var nameT func(string) string - if is.checkPrefix { - nameT = archive.FilterRefPrefix(is.namePrefix) - } else { - nameT = archive.AddRefPrefix(is.namePrefix) - } - name := imageName(desc.Annotations, nameT) - switch refType { - case "name": - if name == "" { - return images.Image{}, fmt.Errorf("no image name: %w", errdefs.ErrNotFound) + // If import ref type, store references from annotation or prefix + if refSource, ok := desc.Annotations["io.containerd.import.ref-source"]; ok { + switch refSource { + case "annotation": + for _, ref := range is.extraReferences { + // Only use prefix references for annotation matching + if !ref.IsPrefix { + continue + } + + var nameT func(string) string + if ref.AllowOverwrite { + nameT = archive.AddRefPrefix(ref.Name) + } else { + nameT = archive.FilterRefPrefix(ref.Name) + } + name := imageName(desc.Annotations, nameT) + + if name == "" { + // If digested, add digest reference + if ref.AddDigest { + imgs = append(imgs, images.Image{ + Name: fmt.Sprintf("%s@%s", ref.Name, desc.Digest), + Target: desc, + Labels: is.imageLabels, + }) + } + continue + } + + imgs = append(imgs, images.Image{ + Name: name, + Target: desc, + Labels: is.imageLabels, + }) + + // If a named reference was found and SkipNamedDigest is true, do + // not use this reference + if ref.AddDigest && !ref.SkipNamedDigest { + imgs = append(imgs, images.Image{ + Name: fmt.Sprintf("%s@%s", ref.Name, desc.Digest), + Target: desc, + Labels: is.imageLabels, + }) + } } - img.Name = name - case "digest": - if !is.digestRefs || (!is.alwaysDigest && name != "") { - return images.Image{}, fmt.Errorf("no digest refs: %w", errdefs.ErrNotFound) - } - img.Name = fmt.Sprintf("%s@%s", is.namePrefix, desc.Digest) default: - return images.Image{}, fmt.Errorf("ref type not supported: %w", errdefs.ErrInvalidArgument) + return nil, fmt.Errorf("ref source not supported: %w", errdefs.ErrInvalidArgument) + } + delete(desc.Annotations, "io.containerd.import.ref-source") + } else { + if is.imageName != "" { + imgs = append(imgs, images.Image{ + Name: is.imageName, + Target: desc, + Labels: is.imageLabels, + }) + } + + // If extra references, store all complete references (skip prefixes) + for _, ref := range is.extraReferences { + if ref.IsPrefix { + continue + } + name := ref.Name + if ref.AddDigest { + name = fmt.Sprintf("%s@%s", name, desc.Digest) + } + imgs = append(imgs, images.Image{ + Name: name, + Target: desc, + Labels: is.imageLabels, + }) } - delete(desc.Annotations, "io.containerd.import.ref-type") - } else if img.Name == "" { - // No valid image combination found - return images.Image{}, fmt.Errorf("no image name found: %w", errdefs.ErrNotFound) } - for { - if created, err := store.Create(ctx, img); err != nil { + if len(imgs) == 0 { + return nil, fmt.Errorf("no image name found: %w", errdefs.ErrNotFound) + } + + for i := 0; i < len(imgs); { + if created, err := store.Create(ctx, imgs[i]); err != nil { if !errdefs.IsAlreadyExists(err) { - return images.Image{}, err + return nil, err } - updated, err := store.Update(ctx, img) + updated, err := store.Update(ctx, imgs[i]) if err != nil { // if image was removed, try create again if errdefs.IsNotFound(err) { + // Keep trying same image continue } - return images.Image{}, err + return nil, err } - img = updated + imgs[i] = updated } else { - img = created + imgs[i] = created } - return img, nil + i++ } + + return imgs, nil } func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, error) { @@ -239,16 +345,13 @@ func (is *Store) UnpackPlatforms() []unpack.Platform { func (is *Store) MarshalAny(context.Context, streaming.StreamCreator) (typeurl.Any, error) { //unpack.Platform s := &transfertypes.ImageStore{ - Name: is.imageName, - Labels: is.imageLabels, - ManifestLimit: uint32(is.manifestLimit), - AllMetadata: is.allMetadata, - Platforms: platformsToProto(is.platforms), - Prefix: is.namePrefix, - CheckPrefix: is.checkPrefix, - DigestRefs: is.digestRefs, - AlwaysDigest: is.alwaysDigest, - Unpacks: unpackToProto(is.unpacks), + Name: is.imageName, + Labels: is.imageLabels, + ManifestLimit: uint32(is.manifestLimit), + AllMetadata: is.allMetadata, + Platforms: platformsToProto(is.platforms), + ExtraReferences: referencesToProto(is.extraReferences), + Unpacks: unpackToProto(is.unpacks), } return typeurl.MarshalAny(s) } @@ -264,10 +367,7 @@ func (is *Store) UnmarshalAny(ctx context.Context, sm streaming.StreamGetter, a is.manifestLimit = int(s.ManifestLimit) is.allMetadata = s.AllMetadata is.platforms = platformFromProto(s.Platforms) - is.namePrefix = s.Prefix - is.checkPrefix = s.CheckPrefix - is.digestRefs = s.DigestRefs - is.alwaysDigest = s.AlwaysDigest + is.extraReferences = referencesFromProto(s.ExtraReferences) is.unpacks = unpackFromProto(s.Unpacks) return nil @@ -297,6 +397,33 @@ func platformFromProto(platforms []*types.Platform) []ocispec.Platform { return op } +func referencesToProto(references []Reference) []*transfertypes.ImageReference { + ir := make([]*transfertypes.ImageReference, len(references)) + for i := range references { + r := transfertypes.ImageReference{ + Name: references[i].Name, + IsPrefix: references[i].IsPrefix, + AllowOverwrite: references[i].AllowOverwrite, + AddDigest: references[i].AddDigest, + SkipNamedDigest: references[i].SkipNamedDigest, + } + + ir[i] = &r + } + return ir +} + +func referencesFromProto(references []*transfertypes.ImageReference) []Reference { + or := make([]Reference, len(references)) + for i := range references { + or[i].Name = references[i].Name + or[i].IsPrefix = references[i].IsPrefix + or[i].AllowOverwrite = references[i].AllowOverwrite + or[i].AddDigest = references[i].AddDigest + or[i].SkipNamedDigest = references[i].SkipNamedDigest + } + return or +} func unpackToProto(uc []UnpackConfiguration) []*transfertypes.UnpackConfiguration { auc := make([]*transfertypes.UnpackConfiguration, len(uc)) for i := range uc { @@ -326,15 +453,23 @@ func unpackFromProto(auc []*transfertypes.UnpackConfiguration) []UnpackConfigura return uc } -func imageName(annotations map[string]string, ociCleanup func(string) string) string { +func imageName(annotations map[string]string, cleanup func(string) string) string { name := annotations[images.AnnotationImageName] if name != "" { + if cleanup != nil { + // containerd reference name should be full reference and not + // modified, if it is incomplete or does not match a specified + // prefix, do not use the reference + if cleanName := cleanup(name); cleanName != name { + name = "" + } + } return name } name = annotations[ocispec.AnnotationRefName] if name != "" { - if ociCleanup != nil { - name = ociCleanup(name) + if cleanup != nil { + name = cleanup(name) } } return name diff --git a/pkg/transfer/image/imagestore_test.go b/pkg/transfer/image/imagestore_test.go new file mode 100644 index 000000000..fbca0e2c3 --- /dev/null +++ b/pkg/transfer/image/imagestore_test.go @@ -0,0 +1,275 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package image + +import ( + "context" + "errors" + "testing" + + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func TestStore(t *testing.T) { + for _, testCase := range []struct { + Name string + ImageStore *Store + // Annotations are the different references annotations to run the test with, + // the possible values: + // - "OCI": Uses the OCI defined annotation "org.opencontainers.image.ref.name" + // This annotation may be a full reference or tag only + // - "containerd": Uses the containerd defined annotation "io.containerd.image.name" + // This annotation is always a full reference as used by containerd + // - "Annotation": Sets the annotation flag but does not set a reference annotation + // Use this case to test the default where no reference is provided + // - "NoAnnotation": Does not set the annotation flag + // Use this case to test storing of the index images by reference + Annotations []string + ImageName string + Images []string + Err error + }{ + { + Name: "Prefix", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/image", + IsPrefix: true, + }, + }, + }, + Annotations: []string{"OCI", "containerd"}, + ImageName: "registry.test/image:latest", + Images: []string{"registry.test/image:latest"}, + }, + { + Name: "Overwrite", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "placeholder", + IsPrefix: true, + AllowOverwrite: true, + }, + }, + }, + Annotations: []string{"OCI", "containerd"}, + ImageName: "registry.test/image:latest", + Images: []string{"registry.test/image:latest"}, + }, + { + Name: "TagOnly", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/image", + IsPrefix: true, + }, + }, + }, + Annotations: []string{"OCI"}, + ImageName: "latest", + Images: []string{"registry.test/image:latest"}, + }, + { + Name: "AddDigest", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/base", + IsPrefix: true, + AddDigest: true, + }, + }, + }, + Annotations: []string{"Annotation"}, + Images: []string{"registry.test/base@"}, + }, + { + Name: "NameAndDigest", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/base", + IsPrefix: true, + AddDigest: true, + }, + }, + }, + Annotations: []string{"OCI"}, + ImageName: "latest", + Images: []string{"registry.test/base:latest", "registry.test/base@"}, + }, + { + Name: "NameSkipDigest", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/base", + IsPrefix: true, + AddDigest: true, + SkipNamedDigest: true, + }, + }, + }, + Annotations: []string{"OCI"}, + ImageName: "latest", + Images: []string{"registry.test/base:latest"}, + }, + { + Name: "OverwriteNameDigest", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "base-name", + IsPrefix: true, + AllowOverwrite: true, + AddDigest: true, + }, + }, + }, + Annotations: []string{"OCI", "containerd"}, + ImageName: "registry.test/base:latest", + Images: []string{"registry.test/base:latest", "base-name@"}, + }, + { + Name: "OverwriteNameSkipDigest", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "base-name", + IsPrefix: true, + AllowOverwrite: true, + AddDigest: true, + SkipNamedDigest: true, + }, + }, + }, + Annotations: []string{"OCI", "containerd"}, + ImageName: "registry.test/base:latest", + Images: []string{"registry.test/base:latest"}, + }, + { + Name: "ReferenceNotFound", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.test/image", + IsPrefix: true, + }, + }, + }, + Annotations: []string{"OCI", "containerd"}, + ImageName: "registry.test/base:latest", + Err: errdefs.ErrNotFound, + }, + { + Name: "NoReference", + ImageStore: &Store{}, + Annotations: []string{"Annotation", "NoAnnotation"}, + Err: errdefs.ErrNotFound, + }, + { + Name: "ImageName", + ImageStore: &Store{ + imageName: "registry.test/index:latest", + }, + Annotations: []string{"NoAnnotation"}, + Images: []string{"registry.test/index:latest"}, + }, + } { + testCase := testCase + for _, a := range testCase.Annotations { + name := testCase.Name + "_" + a + dgst := digest.Canonical.FromString(name) + desc := ocispec.Descriptor{ + Digest: dgst, + Annotations: map[string]string{}, + } + expected := make([]string, len(testCase.Images)) + for i, img := range testCase.Images { + if img[len(img)-1] == '@' { + img = img + dgst.String() + } + expected[i] = img + } + switch a { + case "containerd": + desc.Annotations["io.containerd.import.ref-source"] = "annotation" + desc.Annotations[images.AnnotationImageName] = testCase.ImageName + case "OCI": + desc.Annotations["io.containerd.import.ref-source"] = "annotation" + desc.Annotations[ocispec.AnnotationRefName] = testCase.ImageName + case "Annotation": + desc.Annotations["io.containerd.import.ref-source"] = "annotation" + } + t.Run(name, func(t *testing.T) { + imgs, err := testCase.ImageStore.Store(context.Background(), desc, nopImageStore{}) + if err != nil { + if testCase.Err == nil { + t.Fatal(err) + } + if !errors.Is(err, testCase.Err) { + t.Fatalf("unexpected error %v: expeceted %v", err, testCase.Err) + } + return + } else if testCase.Err != nil { + t.Fatalf("succeeded but expected error: %v", testCase.Err) + } + + if len(imgs) != len(expected) { + t.Fatalf("mismatched array length\nexpected:\n\t%v\nactual\n\t%v", expected, imgs) + } + for i, name := range expected { + if imgs[i].Name != name { + t.Fatalf("wrong image name %q, expected %q", imgs[i].Name, name) + } + if imgs[i].Target.Digest != dgst { + t.Fatalf("wrong image digest %s, expected %s", imgs[i].Target.Digest, dgst) + } + } + }) + } + + } +} + +type nopImageStore struct{} + +func (nopImageStore) Get(ctx context.Context, name string) (images.Image, error) { + return images.Image{}, errdefs.ErrNotFound +} + +func (nopImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { + return nil, nil +} + +func (nopImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { + return image, nil +} + +func (nopImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { + return image, nil +} + +func (nopImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + return nil +} diff --git a/pkg/transfer/local/import.go b/pkg/transfer/local/import.go index 676873ced..5da67ff92 100644 --- a/pkg/transfer/local/import.go +++ b/pkg/transfer/local/import.go @@ -23,6 +23,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/transfer" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -67,14 +68,8 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima } for _, m := range idx.Manifests { - m1 := m - m1.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-type": "name"}) - descriptors = append(descriptors, m1) - - // If add digest references, add twice - m2 := m - m2.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-type": "digest"}) - descriptors = append(descriptors, m2) + m.Annotations = mergeMap(m.Annotations, map[string]string{"io.containerd.import.ref-source": "annotation"}) + descriptors = append(descriptors, m) } return idx.Manifests, nil @@ -85,23 +80,27 @@ func (ts *localTransferService) importStream(ctx context.Context, i transfer.Ima } if err := images.WalkNotEmpty(ctx, handler, index); err != nil { + // TODO: Handle Not Empty as a special case on the input return err } for _, desc := range descriptors { - img, err := is.Store(ctx, desc, ts.images) + imgs, err := is.Store(ctx, desc, ts.images) if err != nil { if errdefs.IsNotFound(err) { + log.G(ctx).Infof("No images store for %s", desc.Digest) continue } return err } if tops.Progress != nil { - tops.Progress(transfer.Progress{ - Event: "saved", - Name: img.Name, - }) + for _, img := range imgs { + tops.Progress(transfer.Progress{ + Event: "saved", + Name: img.Name, + }) + } } } diff --git a/pkg/transfer/local/pull.go b/pkg/transfer/local/pull.go index 44cbf30a7..86ee8140a 100644 --- a/pkg/transfer/local/pull.go +++ b/pkg/transfer/local/pull.go @@ -198,17 +198,18 @@ func (ts *localTransferService) pull(ctx context.Context, ir transfer.ImageFetch } } - img, err := is.Store(ctx, desc, ts.images) + imgs, err := is.Store(ctx, desc, ts.images) if err != nil { return err } if tops.Progress != nil { - tops.Progress(transfer.Progress{ - Event: "saved", - Name: img.Name, - //Digest: img.Target.Digest.String(), - }) + for _, img := range imgs { + tops.Progress(transfer.Progress{ + Event: "saved", + Name: img.Name, + }) + } } if tops.Progress != nil { diff --git a/pkg/transfer/transfer.go b/pkg/transfer/transfer.go index 0440facf7..9bb22d30a 100644 --- a/pkg/transfer/transfer.go +++ b/pkg/transfer/transfer.go @@ -57,10 +57,11 @@ type ImageFilterer interface { ImageFilter(images.HandlerFunc, content.Store) images.HandlerFunc } -// ImageStorer is a type which is capable of storing an image to -// for a provided descriptor +// ImageStorer is a type which is capable of storing images for +// the provided descriptor. The descriptor may be any type of manifest +// including an index with multiple image references. type ImageStorer interface { - Store(context.Context, ocispec.Descriptor, images.Store) (images.Image, error) + Store(context.Context, ocispec.Descriptor, images.Store) ([]images.Image, error) } // ImageGetter is type which returns an image from an image store