From d1627e3c715497decb427ddcd6bd43d17f505992 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 17 Aug 2022 15:32:01 -0700 Subject: [PATCH] Add basic import and export handlers Signed-off-by: Derek McGowan --- api/next.pb.txt | 28 ++++ api/types/transfer/import.pb.go | 226 +++++++++++++++++++++++++++++++ api/types/transfer/import.proto | 29 ++++ pkg/transfer/archive/exporter.go | 87 ++++++++++++ pkg/transfer/archive/importer.go | 76 +++++++++++ pkg/transfer/transfer.go | 11 ++ 6 files changed, 457 insertions(+) create mode 100644 api/types/transfer/import.pb.go create mode 100644 api/types/transfer/import.proto create mode 100644 pkg/transfer/archive/exporter.go create mode 100644 pkg/transfer/archive/importer.go diff --git a/api/next.pb.txt b/api/next.pb.txt index 9fab4146d..de8c683ea 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -5904,6 +5904,34 @@ file { } syntax: "proto3" } +file { + name: "github.com/containerd/containerd/api/types/transfer/import.proto" + package: "containerd.v1.types" + message_type { + name: "ImageImportStream" + field { + name: "stream" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "stream" + } + } + message_type { + name: "ImageExportStream" + field { + name: "stream" + number: 1 + label: LABEL_OPTIONAL + type: TYPE_STRING + json_name: "stream" + } + } + options { + go_package: "github.com/containerd/containerd/api/types/transfer" + } + syntax: "proto3" +} file { name: "github.com/containerd/containerd/api/types/transfer/registry.proto" package: "containerd.v1.types" diff --git a/api/types/transfer/import.pb.go b/api/types/transfer/import.pb.go new file mode 100644 index 000000000..70df2fa0c --- /dev/null +++ b/api/types/transfer/import.pb.go @@ -0,0 +1,226 @@ +// +//Copyright The containerd Authors. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.20.1 +// source: github.com/containerd/containerd/api/types/transfer/import.proto + +package transfer + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ImageImportStream struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Stream string `protobuf:"bytes,1,opt,name=stream,proto3" json:"stream,omitempty"` +} + +func (x *ImageImportStream) Reset() { + *x = ImageImportStream{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ImageImportStream) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImageImportStream) ProtoMessage() {} + +func (x *ImageImportStream) ProtoReflect() protoreflect.Message { + mi := &file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImageImportStream.ProtoReflect.Descriptor instead. +func (*ImageImportStream) Descriptor() ([]byte, []int) { + return file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescGZIP(), []int{0} +} + +func (x *ImageImportStream) GetStream() string { + if x != nil { + return x.Stream + } + return "" +} + +type ImageExportStream struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Stream string `protobuf:"bytes,1,opt,name=stream,proto3" json:"stream,omitempty"` +} + +func (x *ImageExportStream) Reset() { + *x = ImageExportStream{} + if protoimpl.UnsafeEnabled { + mi := &file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ImageExportStream) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImageExportStream) ProtoMessage() {} + +func (x *ImageExportStream) ProtoReflect() protoreflect.Message { + mi := &file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImageExportStream.ProtoReflect.Descriptor instead. +func (*ImageExportStream) Descriptor() ([]byte, []int) { + return file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescGZIP(), []int{1} +} + +func (x *ImageExportStream) GetStream() string { + if x != nil { + return x.Stream + } + return "" +} + +var File_github_com_containerd_containerd_api_types_transfer_import_proto protoreflect.FileDescriptor + +var file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDesc = []byte{ + 0x0a, 0x40, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x13, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x76, + 0x31, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x22, 0x2b, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, + 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x22, 0x2b, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, 0x78, 0x70, + 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescOnce sync.Once + file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescData = file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDesc +) + +func file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescGZIP() []byte { + file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescOnce.Do(func() { + file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescData = protoimpl.X.CompressGZIP(file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescData) + }) + return file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDescData +} + +var file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_github_com_containerd_containerd_api_types_transfer_import_proto_goTypes = []interface{}{ + (*ImageImportStream)(nil), // 0: containerd.v1.types.ImageImportStream + (*ImageExportStream)(nil), // 1: containerd.v1.types.ImageExportStream +} +var file_github_com_containerd_containerd_api_types_transfer_import_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_github_com_containerd_containerd_api_types_transfer_import_proto_init() } +func file_github_com_containerd_containerd_api_types_transfer_import_proto_init() { + if File_github_com_containerd_containerd_api_types_transfer_import_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ImageImportStream); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ImageExportStream); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_github_com_containerd_containerd_api_types_transfer_import_proto_goTypes, + DependencyIndexes: file_github_com_containerd_containerd_api_types_transfer_import_proto_depIdxs, + MessageInfos: file_github_com_containerd_containerd_api_types_transfer_import_proto_msgTypes, + }.Build() + File_github_com_containerd_containerd_api_types_transfer_import_proto = out.File + file_github_com_containerd_containerd_api_types_transfer_import_proto_rawDesc = nil + file_github_com_containerd_containerd_api_types_transfer_import_proto_goTypes = nil + file_github_com_containerd_containerd_api_types_transfer_import_proto_depIdxs = nil +} diff --git a/api/types/transfer/import.proto b/api/types/transfer/import.proto new file mode 100644 index 000000000..fd36a5176 --- /dev/null +++ b/api/types/transfer/import.proto @@ -0,0 +1,29 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +syntax = "proto3"; + +package containerd.v1.types; + +option go_package = "github.com/containerd/containerd/api/types/transfer"; + +message ImageImportStream { + string stream = 1; +} + +message ImageExportStream { + string stream = 1; +} \ No newline at end of file diff --git a/pkg/transfer/archive/exporter.go b/pkg/transfer/archive/exporter.go new file mode 100644 index 000000000..cf619ae42 --- /dev/null +++ b/pkg/transfer/archive/exporter.go @@ -0,0 +1,87 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package archive + +import ( + "context" + "io" + + transferapi "github.com/containerd/containerd/api/types/transfer" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/pkg/streaming" + tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" + "github.com/containerd/typeurl" +) + +// NewImageImportStream returns a image importer via tar stream +// TODO: Add import options +func NewImageExportStream(stream io.WriteCloser) *ImageExportStream { + return &ImageExportStream{ + stream: stream, + } +} + +type ImageExportStream struct { + stream io.WriteCloser +} + +func (iis *ImageExportStream) ExportStream(context.Context) (io.WriteCloser, error) { + return iis.stream, nil +} + +func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.StreamManager) (typeurl.Any, error) { + // TODO: Unique stream ID + sid := "randomid" + // TODO: Should this API be create instead of get + stream, err := sm.Get(ctx, sid) + if err != nil { + return nil, err + } + + // Receive stream and copy to writer + go func() { + if _, err := io.Copy(iis.stream, tstreaming.ReceiveStream(ctx, stream)); err != nil { + log.G(ctx).WithError(err).WithField("streamid", sid).Errorf("error copying stream") + } + iis.stream.Close() + }() + + s := &transferapi.ImageExportStream{ + Stream: sid, + } + + return typeurl.MarshalAny(s) +} + +func (iis *ImageExportStream) UnmarshalAny(ctx context.Context, sm streaming.StreamManager, any typeurl.Any) error { + var s transferapi.ImageExportStream + if err := typeurl.UnmarshalTo(any, &s); err != nil { + return err + } + + stream, err := sm.Get(ctx, s.Stream) + if err != nil { + return err + } + + r, w := io.Pipe() + + tstreaming.SendStream(ctx, r, stream) + iis.stream = w + + return nil +} diff --git a/pkg/transfer/archive/importer.go b/pkg/transfer/archive/importer.go new file mode 100644 index 000000000..d1170b485 --- /dev/null +++ b/pkg/transfer/archive/importer.go @@ -0,0 +1,76 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package archive + +import ( + "context" + "io" + + transferapi "github.com/containerd/containerd/api/types/transfer" + "github.com/containerd/containerd/pkg/streaming" + tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" + "github.com/containerd/typeurl" +) + +// NewImageImportStream returns a image importer via tar stream +// TODO: Add import options +func NewImageImportStream(stream io.Reader) *ImageImportStream { + return &ImageImportStream{ + stream: stream, + } +} + +type ImageImportStream struct { + stream io.Reader +} + +func (iis *ImageImportStream) ImportStream(context.Context) (io.Reader, error) { + return iis.stream, nil +} + +func (iis *ImageImportStream) MarshalAny(ctx context.Context, sm streaming.StreamManager) (typeurl.Any, error) { + // TODO: Unique stream ID + sid := "randomid" + // TODO: Should this API be create instead of get + stream, err := sm.Get(ctx, sid) + if err != nil { + return nil, err + } + tstreaming.SendStream(ctx, iis.stream, stream) + + s := &transferapi.ImageImportStream{ + Stream: sid, + } + + return typeurl.MarshalAny(s) +} + +func (iis *ImageImportStream) UnmarshalAny(ctx context.Context, sm streaming.StreamManager, any typeurl.Any) error { + var s transferapi.ImageImportStream + if err := typeurl.UnmarshalTo(any, &s); err != nil { + return err + } + + stream, err := sm.Get(ctx, s.Stream) + if err != nil { + return err + } + + iis.stream = tstreaming.ReceiveStream(ctx, stream) + + return nil +} diff --git a/pkg/transfer/transfer.go b/pkg/transfer/transfer.go index 5a8bb6581..5cecc30a5 100644 --- a/pkg/transfer/transfer.go +++ b/pkg/transfer/transfer.go @@ -50,6 +50,17 @@ type ImageStorer interface { Store(context.Context, ocispec.Descriptor) (images.Image, error) } +// ImageImportStreamer returns an import streamer based on OCI or +// Docker image tar archives. The stream should be a raw tar stream +// and without compression. +type ImageImportStreamer interface { + ImportStream(context.Context) (io.Reader, error) +} + +type ImageExportStreamer interface { + ExportStream(context.Context) (io.WriteCloser, error) +} + type ImageUnpacker interface { // TODO: Or unpack options? UnpackPlatforms() []unpack.Platform