From b9d7eae1ad4d97af2a7c789cb6d2e6dc1f3afe51 Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Wed, 4 Jan 2023 15:54:04 +0800 Subject: [PATCH 1/4] feat(api): add fields to ImageExportStream Signed-off-by: Jian Zeng --- api/next.pb.txt | 37 +++++++++ api/types/transfer/importexport.pb.go | 109 ++++++++++++++++++++------ api/types/transfer/importexport.proto | 13 +++ 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/api/next.pb.txt b/api/next.pb.txt index c37a8c9d9..708acecb9 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -6857,6 +6857,7 @@ file { file { name: "github.com/containerd/containerd/api/types/transfer/importexport.proto" package: "containerd.types.transfer" + dependency: "github.com/containerd/containerd/api/types/platform.proto" message_type { name: "ImageImportStream" field { @@ -6897,6 +6898,42 @@ file { type: TYPE_STRING json_name: "mediaType" } + field { + name: "images" + number: 3 + label: LABEL_REPEATED + type: TYPE_STRING + json_name: "images" + } + field { + name: "platforms" + number: 4 + label: LABEL_REPEATED + type: TYPE_MESSAGE + type_name: ".containerd.types.Platform" + json_name: "platforms" + } + field { + name: "all_platforms" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "allPlatforms" + } + field { + name: "skip_docker_manifest" + number: 6 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "skipDockerManifest" + } + field { + name: "skip_non_distributable" + number: 7 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "skipNonDistributable" + } } options { go_package: "github.com/containerd/containerd/api/types/transfer" diff --git a/api/types/transfer/importexport.pb.go b/api/types/transfer/importexport.pb.go index ca24b3cb3..83892ff36 100644 --- a/api/types/transfer/importexport.pb.go +++ b/api/types/transfer/importexport.pb.go @@ -22,6 +22,7 @@ package transfer import ( + types "github.com/containerd/containerd/api/types" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -111,6 +112,16 @@ type ImageExportStream struct { // The binary data is expected to be a raw tar stream. Stream string `protobuf:"bytes,1,opt,name=stream,proto3" json:"stream,omitempty"` MediaType string `protobuf:"bytes,2,opt,name=media_type,json=mediaType,proto3" json:"media_type,omitempty"` + // The image names + Images []string `protobuf:"bytes,3,rep,name=images,proto3" json:"images,omitempty"` + // The specified platforms + Platforms []*types.Platform `protobuf:"bytes,4,rep,name=platforms,proto3" json:"platforms,omitempty"` + // Whether to include all platforms + AllPlatforms bool `protobuf:"varint,5,opt,name=all_platforms,json=allPlatforms,proto3" json:"all_platforms,omitempty"` + // Skips the creation of the Docker compatible manifest.json file + SkipDockerManifest bool `protobuf:"varint,6,opt,name=skip_docker_manifest,json=skipDockerManifest,proto3" json:"skip_docker_manifest,omitempty"` + // Excludes non-distributable blobs such as Windows base layers. + SkipNonDistributable bool `protobuf:"varint,7,opt,name=skip_non_distributable,json=skipNonDistributable,proto3" json:"skip_non_distributable,omitempty"` } func (x *ImageExportStream) Reset() { @@ -159,6 +170,41 @@ func (x *ImageExportStream) GetMediaType() string { return "" } +func (x *ImageExportStream) GetImages() []string { + if x != nil { + return x.Images + } + return nil +} + +func (x *ImageExportStream) GetPlatforms() []*types.Platform { + if x != nil { + return x.Platforms + } + return nil +} + +func (x *ImageExportStream) GetAllPlatforms() bool { + if x != nil { + return x.AllPlatforms + } + return false +} + +func (x *ImageExportStream) GetSkipDockerManifest() bool { + if x != nil { + return x.SkipDockerManifest + } + return false +} + +func (x *ImageExportStream) GetSkipNonDistributable() bool { + if x != nil { + return x.SkipNonDistributable + } + return false +} + var File_github_com_containerd_containerd_api_types_transfer_importexport_proto protoreflect.FileDescriptor var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_rawDesc = []byte{ @@ -168,23 +214,40 @@ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_ 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x65, 0x72, 0x22, 0x71, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6d, 0x70, 0x6f, - 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x6f, - 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x22, 0x4a, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, - 0x70, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x66, 0x65, 0x72, 0x1a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x71, + 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, + 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, + 0x0a, 0x06, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, + 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, + 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, + 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x61, 0x6c, 0x6c, 0x50, 0x6c, 0x61, 0x74, + 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x64, 0x6f, + 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x12, 0x73, 0x6b, 0x69, 0x70, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x6b, 0x69, 0x70, 0x5f, + 0x6e, 0x6f, 0x6e, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, + 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x4e, 0x6f, 0x6e, + 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x35, 0x5a, + 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, + 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, + 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -203,13 +266,15 @@ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_goTypes = []interface{}{ (*ImageImportStream)(nil), // 0: containerd.types.transfer.ImageImportStream (*ImageExportStream)(nil), // 1: containerd.types.transfer.ImageExportStream + (*types.Platform)(nil), // 2: containerd.types.Platform } var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: containerd.types.transfer.ImageExportStream.platforms:type_name -> containerd.types.Platform + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_github_com_containerd_containerd_api_types_transfer_importexport_proto_init() } diff --git a/api/types/transfer/importexport.proto b/api/types/transfer/importexport.proto index 54c9db9c0..5675adb20 100644 --- a/api/types/transfer/importexport.proto +++ b/api/types/transfer/importexport.proto @@ -20,6 +20,8 @@ package containerd.types.transfer; option go_package = "github.com/containerd/containerd/api/types/transfer"; +import "github.com/containerd/containerd/api/types/platform.proto"; + message ImageImportStream { // Stream is used to identify the binary input stream for the import operation. // The stream uses the transfer binary stream protocol with the client as the sender. @@ -38,4 +40,15 @@ message ImageExportStream { string stream = 1; string media_type = 2; + + // The image names + repeated string images = 3; + // The specified platforms + repeated types.Platform platforms = 4; + // Whether to include all platforms + bool all_platforms = 5; + // Skips the creation of the Docker compatible manifest.json file + bool skip_docker_manifest = 6; + // Excludes non-distributable blobs such as Windows base layers. + bool skip_non_distributable = 7; } From f6491b0049ae9a0c124b642c77d796677ff48328 Mon Sep 17 00:00:00 2001 From: Jian Zeng Date: Wed, 4 Jan 2023 16:52:30 +0800 Subject: [PATCH 2/4] feat: export images using Transfer api Signed-off-by: Jian Zeng --- cmd/ctr/commands/images/export.go | 81 +++++++++++++++++++++------- pkg/transfer/archive/exporter.go | 87 ++++++++++++++++++++++++++++--- pkg/transfer/local/export.go | 49 +++++++++++++++++ pkg/transfer/local/transfer.go | 2 + pkg/transfer/transfer.go | 5 ++ 5 files changed, 198 insertions(+), 26 deletions(-) create mode 100644 pkg/transfer/local/export.go diff --git a/cmd/ctr/commands/images/export.go b/cmd/ctr/commands/images/export.go index d4e9be9b2..269267033 100644 --- a/cmd/ctr/commands/images/export.go +++ b/cmd/ctr/commands/images/export.go @@ -22,11 +22,15 @@ import ( "io" "os" - "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images/archive" - "github.com/containerd/containerd/platforms" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/urfave/cli" + + "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/pkg/transfer" + tarchive "github.com/containerd/containerd/pkg/transfer/archive" + "github.com/containerd/containerd/pkg/transfer/image" + "github.com/containerd/containerd/platforms" ) var exportCommand = cli.Command{ @@ -58,6 +62,10 @@ When '--all-platforms' is given all images in a manifest list must be available. Name: "all-platforms", Usage: "Exports content from all platforms", }, + cli.BoolTFlag{ + Name: "local", + Usage: "run export locally rather than through transfer API", + }, }, Action: func(context *cli.Context) error { var ( @@ -69,6 +77,56 @@ When '--all-platforms' is given all images in a manifest list must be available. return errors.New("please provide both an output filename and an image reference to export") } + client, ctx, cancel, err := commands.NewClient(context) + if err != nil { + return err + } + defer cancel() + + var w io.WriteCloser + if out == "-" { + w = os.Stdout + } else { + w, err = os.Create(out) + if err != nil { + return err + } + } + defer w.Close() + + if !context.BoolT("local") { + pf, done := ProgressHandler(ctx, os.Stdout) + defer done() + + var specified []ocispec.Platform + if pss := context.StringSlice("platform"); len(pss) > 0 { + for _, ps := range pss { + p, err := platforms.Parse(ps) + if err != nil { + return fmt.Errorf("invalid platform %q: %w", ps, err) + } + specified = append(specified, p) + } + } + + err := client.Transfer(ctx, + image.NewStore(""), // a dummy image store + tarchive.NewImageExportStream(w, "", tarchive.ExportOptions{ + Images: images, + Platforms: specified, + AllPlatforms: context.Bool("all-platforms"), + SkipNonDistributable: context.Bool("skip-non-distributable"), + SkipDockerManifest: context.Bool("skip-manifest-json"), + }), + transfer.WithProgress(pf), + ) + if err != nil { + return err + } + + return nil + } + if pss := context.StringSlice("platform"); len(pss) > 0 { var all []ocispec.Platform for _, ps := range pss { @@ -95,28 +153,11 @@ When '--all-platforms' is given all images in a manifest list must be available. exportOpts = append(exportOpts, archive.WithSkipNonDistributableBlobs()) } - client, ctx, cancel, err := commands.NewClient(context) - if err != nil { - return err - } - defer cancel() - is := client.ImageService() for _, img := range images { exportOpts = append(exportOpts, archive.WithImage(is, img)) } - var w io.WriteCloser - if out == "-" { - w = os.Stdout - } else { - w, err = os.Create(out) - if err != nil { - return err - } - } - defer w.Close() - return client.Export(ctx, w, exportOpts...) }, } diff --git a/pkg/transfer/archive/exporter.go b/pkg/transfer/archive/exporter.go index a5ef20872..044e76fd2 100644 --- a/pkg/transfer/archive/exporter.go +++ b/pkg/transfer/archive/exporter.go @@ -20,12 +20,19 @@ import ( "context" "io" + "github.com/containerd/typeurl/v2" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/containerd/containerd/api/types" transfertypes "github.com/containerd/containerd/api/types/transfer" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/streaming" "github.com/containerd/containerd/pkg/transfer/plugins" tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" - "github.com/containerd/typeurl/v2" + "github.com/containerd/containerd/platforms" ) func init() { @@ -34,24 +41,65 @@ func init() { plugins.Register(&transfertypes.ImageImportStream{}, &ImageImportStream{}) } -// NewImageExportStream returns a image importer via tar stream -// TODO: Add export options -func NewImageExportStream(stream io.WriteCloser, mediaType string) *ImageExportStream { +type ExportOptions struct { + Images []string + Platforms []v1.Platform + AllPlatforms bool + SkipDockerManifest bool + SkipNonDistributable bool +} + +// NewImageExportStream returns an image exporter via tar stream +func NewImageExportStream(stream io.WriteCloser, mediaType string, opts ExportOptions) *ImageExportStream { return &ImageExportStream{ stream: stream, mediaType: mediaType, + + images: opts.Images, + platforms: opts.Platforms, + allPlatforms: opts.AllPlatforms, + skipDockerManifest: opts.SkipDockerManifest, + skipNonDistributable: opts.SkipNonDistributable, } } type ImageExportStream struct { stream io.WriteCloser mediaType string + + images []string + platforms []v1.Platform + allPlatforms bool + skipDockerManifest bool + skipNonDistributable bool } func (iis *ImageExportStream) ExportStream(context.Context) (io.WriteCloser, string, error) { return iis.stream, iis.mediaType, nil } +func (iis *ImageExportStream) Export(ctx context.Context, is images.Store, cs content.Store) error { + var opts []archive.ExportOpt + for _, img := range iis.images { + opts = append(opts, archive.WithImage(is, img)) + } + if len(iis.platforms) > 0 { + opts = append(opts, archive.WithPlatform(platforms.Ordered(iis.platforms...))) + } else { + opts = append(opts, archive.WithPlatform(platforms.DefaultStrict())) + } + if iis.allPlatforms { + opts = append(opts, archive.WithAllPlatforms()) + } + if iis.skipDockerManifest { + opts = append(opts, archive.WithSkipDockerManifest()) + } + if iis.skipNonDistributable { + opts = append(opts, archive.WithSkipNonDistributableBlobs()) + } + return archive.Export(ctx, cs, iis.stream, opts...) +} + func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) { sid := tstreaming.GenerateID("export") stream, err := sm.Create(ctx, sid) @@ -67,9 +115,22 @@ func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.Strea iis.stream.Close() }() + var specified []*types.Platform + for _, p := range iis.platforms { + specified = append(specified, &types.Platform{ + OS: p.OS, + Architecture: p.Architecture, + Variant: p.Variant, + }) + } s := &transfertypes.ImageExportStream{ - Stream: sid, - MediaType: iis.mediaType, + Stream: sid, + MediaType: iis.mediaType, + Images: iis.images, + Platforms: specified, + AllPlatforms: iis.allPlatforms, + SkipDockerManifest: iis.skipDockerManifest, + SkipNonDistributable: iis.skipNonDistributable, } return typeurl.MarshalAny(s) @@ -87,8 +148,22 @@ func (iis *ImageExportStream) UnmarshalAny(ctx context.Context, sm streaming.Str return err } + var specified []v1.Platform + for _, p := range s.Platforms { + specified = append(specified, v1.Platform{ + OS: p.OS, + Architecture: p.Architecture, + Variant: p.Variant, + }) + } + iis.stream = tstreaming.WriteByteStream(ctx, stream) iis.mediaType = s.MediaType + iis.images = s.Images + iis.platforms = specified + iis.allPlatforms = s.AllPlatforms + iis.skipDockerManifest = s.SkipDockerManifest + iis.skipNonDistributable = s.SkipNonDistributable return nil } diff --git a/pkg/transfer/local/export.go b/pkg/transfer/local/export.go new file mode 100644 index 000000000..6c7cff926 --- /dev/null +++ b/pkg/transfer/local/export.go @@ -0,0 +1,49 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package local + +import ( + "context" + + "github.com/containerd/containerd/pkg/transfer" +) + +func (ts *localTransferService) exportStream(ctx context.Context, is transfer.ImageExporter, tops *transfer.Config) error { + ctx, done, err := ts.withLease(ctx) + if err != nil { + return err + } + defer done(ctx) + + if tops.Progress != nil { + tops.Progress(transfer.Progress{ + Event: "Exporting", + }) + } + + err = is.Export(ctx, ts.images, ts.content) + if err != nil { + return err + } + + if tops.Progress != nil { + tops.Progress(transfer.Progress{ + Event: "Completed export", + }) + } + return nil +} diff --git a/pkg/transfer/local/transfer.go b/pkg/transfer/local/transfer.go index 0189290dd..d0a555d1c 100644 --- a/pkg/transfer/local/transfer.go +++ b/pkg/transfer/local/transfer.go @@ -72,6 +72,8 @@ func (ts *localTransferService) Transfer(ctx context.Context, src interface{}, d switch d := dest.(type) { case transfer.ImagePusher: return ts.push(ctx, s, d, topts) + case transfer.ImageExporter: + return ts.exportStream(ctx, d, topts) } case transfer.ImageImporter: switch d := dest.(type) { diff --git a/pkg/transfer/transfer.go b/pkg/transfer/transfer.go index a9c4fcd73..f87a99061 100644 --- a/pkg/transfer/transfer.go +++ b/pkg/transfer/transfer.go @@ -69,6 +69,11 @@ type ImageGetter interface { Get(context.Context, images.Store) (images.Image, error) } +// ImageExporter exports images to a writer +type ImageExporter interface { + Export(ctx context.Context, is images.Store, cs content.Store) error +} + // ImageImporter imports an image into a content store type ImageImporter interface { Import(context.Context, content.Store) (ocispec.Descriptor, error) From e2283edefb68377e9d485c10b919dab73510f2b5 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 28 Feb 2023 22:07:19 -0800 Subject: [PATCH 3/4] [transfer] update export API types Signed-off-by: Derek McGowan --- api/next.pb.txt | 19 +++------ api/types/transfer/importexport.pb.go | 61 ++++++++++++--------------- api/types/transfer/importexport.proto | 10 ++--- 3 files changed, 36 insertions(+), 54 deletions(-) diff --git a/api/next.pb.txt b/api/next.pb.txt index 708acecb9..ec7fd3da4 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -6898,16 +6898,9 @@ file { type: TYPE_STRING json_name: "mediaType" } - field { - name: "images" - number: 3 - label: LABEL_REPEATED - type: TYPE_STRING - json_name: "images" - } field { name: "platforms" - number: 4 + number: 3 label: LABEL_REPEATED type: TYPE_MESSAGE type_name: ".containerd.types.Platform" @@ -6915,21 +6908,21 @@ file { } field { name: "all_platforms" - number: 5 + number: 4 label: LABEL_OPTIONAL type: TYPE_BOOL json_name: "allPlatforms" } field { - name: "skip_docker_manifest" - number: 6 + name: "skip_compatibility_manifest" + number: 5 label: LABEL_OPTIONAL type: TYPE_BOOL - json_name: "skipDockerManifest" + json_name: "skipCompatibilityManifest" } field { name: "skip_non_distributable" - number: 7 + number: 6 label: LABEL_OPTIONAL type: TYPE_BOOL json_name: "skipNonDistributable" diff --git a/api/types/transfer/importexport.pb.go b/api/types/transfer/importexport.pb.go index 83892ff36..a2a48ac15 100644 --- a/api/types/transfer/importexport.pb.go +++ b/api/types/transfer/importexport.pb.go @@ -112,16 +112,14 @@ type ImageExportStream struct { // The binary data is expected to be a raw tar stream. Stream string `protobuf:"bytes,1,opt,name=stream,proto3" json:"stream,omitempty"` MediaType string `protobuf:"bytes,2,opt,name=media_type,json=mediaType,proto3" json:"media_type,omitempty"` - // The image names - Images []string `protobuf:"bytes,3,rep,name=images,proto3" json:"images,omitempty"` // The specified platforms - Platforms []*types.Platform `protobuf:"bytes,4,rep,name=platforms,proto3" json:"platforms,omitempty"` + Platforms []*types.Platform `protobuf:"bytes,3,rep,name=platforms,proto3" json:"platforms,omitempty"` // Whether to include all platforms - AllPlatforms bool `protobuf:"varint,5,opt,name=all_platforms,json=allPlatforms,proto3" json:"all_platforms,omitempty"` + AllPlatforms bool `protobuf:"varint,4,opt,name=all_platforms,json=allPlatforms,proto3" json:"all_platforms,omitempty"` // Skips the creation of the Docker compatible manifest.json file - SkipDockerManifest bool `protobuf:"varint,6,opt,name=skip_docker_manifest,json=skipDockerManifest,proto3" json:"skip_docker_manifest,omitempty"` + SkipCompatibilityManifest bool `protobuf:"varint,5,opt,name=skip_compatibility_manifest,json=skipCompatibilityManifest,proto3" json:"skip_compatibility_manifest,omitempty"` // Excludes non-distributable blobs such as Windows base layers. - SkipNonDistributable bool `protobuf:"varint,7,opt,name=skip_non_distributable,json=skipNonDistributable,proto3" json:"skip_non_distributable,omitempty"` + SkipNonDistributable bool `protobuf:"varint,6,opt,name=skip_non_distributable,json=skipNonDistributable,proto3" json:"skip_non_distributable,omitempty"` } func (x *ImageExportStream) Reset() { @@ -170,13 +168,6 @@ func (x *ImageExportStream) GetMediaType() string { return "" } -func (x *ImageExportStream) GetImages() []string { - if x != nil { - return x.Images - } - return nil -} - func (x *ImageExportStream) GetPlatforms() []*types.Platform { if x != nil { return x.Platforms @@ -191,9 +182,9 @@ func (x *ImageExportStream) GetAllPlatforms() bool { return false } -func (x *ImageExportStream) GetSkipDockerManifest() bool { +func (x *ImageExportStream) GetSkipCompatibilityManifest() bool { if x != nil { - return x.SkipDockerManifest + return x.SkipCompatibilityManifest } return false } @@ -225,29 +216,29 @@ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_ 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x22, 0xa9, 0x02, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x73, 0x22, 0x9f, 0x02, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, - 0x0a, 0x06, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, - 0x69, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x12, 0x38, 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, - 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, - 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, - 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, - 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x61, 0x6c, 0x6c, 0x50, 0x6c, 0x61, 0x74, - 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x30, 0x0a, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x64, 0x6f, - 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x12, 0x73, 0x6b, 0x69, 0x70, 0x44, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x4d, - 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x73, 0x6b, 0x69, 0x70, 0x5f, - 0x6e, 0x6f, 0x6e, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, - 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, 0x6b, 0x69, 0x70, 0x4e, 0x6f, 0x6e, - 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x42, 0x35, 0x5a, - 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, - 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, + 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, 0x5f, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0c, 0x61, 0x6c, 0x6c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x3e, 0x0a, + 0x1b, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x69, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x19, 0x73, 0x6b, 0x69, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x69, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x79, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, + 0x16, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, + 0x6b, 0x69, 0x70, 0x4e, 0x6f, 0x6e, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( diff --git a/api/types/transfer/importexport.proto b/api/types/transfer/importexport.proto index 5675adb20..c18bae1c6 100644 --- a/api/types/transfer/importexport.proto +++ b/api/types/transfer/importexport.proto @@ -41,14 +41,12 @@ message ImageExportStream { string media_type = 2; - // The image names - repeated string images = 3; // The specified platforms - repeated types.Platform platforms = 4; + repeated types.Platform platforms = 3; // Whether to include all platforms - bool all_platforms = 5; + bool all_platforms = 4; // Skips the creation of the Docker compatible manifest.json file - bool skip_docker_manifest = 6; + bool skip_compatibility_manifest = 5; // Excludes non-distributable blobs such as Windows base layers. - bool skip_non_distributable = 7; + bool skip_non_distributable = 6; } From 13bf5565eb614b6221ddd2da2a222e8d715f6fb3 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 28 Feb 2023 22:15:44 -0800 Subject: [PATCH 4/4] [transfer] update export to use image store references Signed-off-by: Derek McGowan --- cmd/ctr/commands/images/export.go | 39 +++--- images/archive/exporter.go | 12 ++ pkg/transfer/archive/exporter.go | 73 ++++++------ pkg/transfer/image/imagestore.go | 22 ++++ pkg/transfer/image/imagestore_test.go | 163 ++++++++++++++++++++++++-- pkg/transfer/local/export.go | 19 ++- pkg/transfer/local/transfer.go | 2 +- pkg/transfer/transfer.go | 8 +- 8 files changed, 274 insertions(+), 64 deletions(-) diff --git a/cmd/ctr/commands/images/export.go b/cmd/ctr/commands/images/export.go index 269267033..b61641d86 100644 --- a/cmd/ctr/commands/images/export.go +++ b/cmd/ctr/commands/images/export.go @@ -98,33 +98,38 @@ When '--all-platforms' is given all images in a manifest list must be available. pf, done := ProgressHandler(ctx, os.Stdout) defer done() - var specified []ocispec.Platform + exportOpts := []tarchive.ExportOpt{} if pss := context.StringSlice("platform"); len(pss) > 0 { for _, ps := range pss { p, err := platforms.Parse(ps) if err != nil { return fmt.Errorf("invalid platform %q: %w", ps, err) } - specified = append(specified, p) + exportOpts = append(exportOpts, tarchive.WithPlatform(p)) } } - - err := client.Transfer(ctx, - image.NewStore(""), // a dummy image store - tarchive.NewImageExportStream(w, "", tarchive.ExportOptions{ - Images: images, - Platforms: specified, - AllPlatforms: context.Bool("all-platforms"), - SkipNonDistributable: context.Bool("skip-non-distributable"), - SkipDockerManifest: context.Bool("skip-manifest-json"), - }), - transfer.WithProgress(pf), - ) - if err != nil { - return err + if context.Bool("all-platforms") { + exportOpts = append(exportOpts, tarchive.WithAllPlatforms) } - return nil + if context.Bool("skip-manifest-json") { + exportOpts = append(exportOpts, tarchive.WithSkipCompatibilityManifest) + } + + if context.Bool("skip-non-distributable") { + exportOpts = append(exportOpts, tarchive.WithSkipNonDistributableBlobs) + } + + storeOpts := []image.StoreOpt{} + for _, img := range images { + storeOpts = append(storeOpts, image.WithExtraReference(img)) + } + + return client.Transfer(ctx, + image.NewStore("", storeOpts...), + tarchive.NewImageExportStream(w, "", exportOpts...), + transfer.WithProgress(pf), + ) } if pss := context.StringSlice("platform"); len(pss) > 0 { diff --git a/images/archive/exporter.go b/images/archive/exporter.go index 40a0a33df..87858a958 100644 --- a/images/archive/exporter.go +++ b/images/archive/exporter.go @@ -89,6 +89,18 @@ func WithImage(is images.Store, name string) ExportOpt { } } +// WithImages adds multiples images to the exported archive. +func WithImages(imgs []images.Image) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + for _, img := range imgs { + img.Target.Annotations = addNameAnnotation(img.Name, img.Target.Annotations) + o.manifests = append(o.manifests, img.Target) + } + + return nil + } +} + // WithManifest adds a manifest to the exported archive. // When names are given they will be set on the manifest in the // exported archive, creating an index record for each name. diff --git a/pkg/transfer/archive/exporter.go b/pkg/transfer/archive/exporter.go index 044e76fd2..80a6d2dc1 100644 --- a/pkg/transfer/archive/exporter.go +++ b/pkg/transfer/archive/exporter.go @@ -41,48 +41,57 @@ func init() { plugins.Register(&transfertypes.ImageImportStream{}, &ImageImportStream{}) } -type ExportOptions struct { - Images []string - Platforms []v1.Platform - AllPlatforms bool - SkipDockerManifest bool - SkipNonDistributable bool +type ExportOpt func(*ImageExportStream) + +func WithPlatform(p v1.Platform) ExportOpt { + return func(s *ImageExportStream) { + s.platforms = append(s.platforms, p) + } +} + +func WithAllPlatforms(s *ImageExportStream) { + s.allPlatforms = true +} + +func WithSkipCompatibilityManifest(s *ImageExportStream) { + s.skipCompatibilityManifest = true +} + +func WithSkipNonDistributableBlobs(s *ImageExportStream) { + s.skipNonDistributable = true } // NewImageExportStream returns an image exporter via tar stream -func NewImageExportStream(stream io.WriteCloser, mediaType string, opts ExportOptions) *ImageExportStream { - return &ImageExportStream{ +func NewImageExportStream(stream io.WriteCloser, mediaType string, opts ...ExportOpt) *ImageExportStream { + s := &ImageExportStream{ stream: stream, mediaType: mediaType, - - images: opts.Images, - platforms: opts.Platforms, - allPlatforms: opts.AllPlatforms, - skipDockerManifest: opts.SkipDockerManifest, - skipNonDistributable: opts.SkipNonDistributable, } + for _, opt := range opts { + opt(s) + } + return s } type ImageExportStream struct { stream io.WriteCloser mediaType string - images []string - platforms []v1.Platform - allPlatforms bool - skipDockerManifest bool - skipNonDistributable bool + platforms []v1.Platform + allPlatforms bool + skipCompatibilityManifest bool + skipNonDistributable bool } func (iis *ImageExportStream) ExportStream(context.Context) (io.WriteCloser, string, error) { return iis.stream, iis.mediaType, nil } -func (iis *ImageExportStream) Export(ctx context.Context, is images.Store, cs content.Store) error { - var opts []archive.ExportOpt - for _, img := range iis.images { - opts = append(opts, archive.WithImage(is, img)) +func (iis *ImageExportStream) Export(ctx context.Context, cs content.Store, imgs []images.Image) error { + opts := []archive.ExportOpt{ + archive.WithImages(imgs), } + if len(iis.platforms) > 0 { opts = append(opts, archive.WithPlatform(platforms.Ordered(iis.platforms...))) } else { @@ -91,7 +100,7 @@ func (iis *ImageExportStream) Export(ctx context.Context, is images.Store, cs co if iis.allPlatforms { opts = append(opts, archive.WithAllPlatforms()) } - if iis.skipDockerManifest { + if iis.skipCompatibilityManifest { opts = append(opts, archive.WithSkipDockerManifest()) } if iis.skipNonDistributable { @@ -124,13 +133,12 @@ func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.Strea }) } s := &transfertypes.ImageExportStream{ - Stream: sid, - MediaType: iis.mediaType, - Images: iis.images, - Platforms: specified, - AllPlatforms: iis.allPlatforms, - SkipDockerManifest: iis.skipDockerManifest, - SkipNonDistributable: iis.skipNonDistributable, + Stream: sid, + MediaType: iis.mediaType, + Platforms: specified, + AllPlatforms: iis.allPlatforms, + SkipCompatibilityManifest: iis.skipCompatibilityManifest, + SkipNonDistributable: iis.skipNonDistributable, } return typeurl.MarshalAny(s) @@ -159,10 +167,9 @@ func (iis *ImageExportStream) UnmarshalAny(ctx context.Context, sm streaming.Str iis.stream = tstreaming.WriteByteStream(ctx, stream) iis.mediaType = s.MediaType - iis.images = s.Images iis.platforms = specified iis.allPlatforms = s.AllPlatforms - iis.skipDockerManifest = s.SkipDockerManifest + iis.skipCompatibilityManifest = s.SkipCompatibilityManifest iis.skipNonDistributable = s.SkipNonDistributable return nil diff --git a/pkg/transfer/image/imagestore.go b/pkg/transfer/image/imagestore.go index 547228a15..b26f25605 100644 --- a/pkg/transfer/image/imagestore.go +++ b/pkg/transfer/image/imagestore.go @@ -325,6 +325,28 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err return store.Get(ctx, is.imageName) } +func (is *Store) Lookup(ctx context.Context, store images.Store) ([]images.Image, error) { + var imgs []images.Image + if is.imageName != "" { + img, err := store.Get(ctx, is.imageName) + if err != nil { + return nil, err + } + imgs = append(imgs, img) + } + for _, ref := range is.extraReferences { + if ref.IsPrefix { + return nil, fmt.Errorf("prefix lookup on export not implemented: %w", errdefs.ErrNotImplemented) + } + img, err := store.Get(ctx, ref.Name) + if err != nil { + return nil, err + } + imgs = append(imgs, img) + } + return imgs, nil +} + func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration { unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks)) for i, uc := range is.unpacks { diff --git a/pkg/transfer/image/imagestore_test.go b/pkg/transfer/image/imagestore_test.go index fbca0e2c3..cefa0c1ef 100644 --- a/pkg/transfer/image/imagestore_test.go +++ b/pkg/transfer/image/imagestore_test.go @@ -19,6 +19,8 @@ package image import ( "context" "errors" + "sort" + "sync" "testing" "github.com/containerd/containerd/errdefs" @@ -222,7 +224,7 @@ func TestStore(t *testing.T) { desc.Annotations["io.containerd.import.ref-source"] = "annotation" } t.Run(name, func(t *testing.T) { - imgs, err := testCase.ImageStore.Store(context.Background(), desc, nopImageStore{}) + imgs, err := testCase.ImageStore.Store(context.Background(), desc, newSimpleImageStore()) if err != nil { if testCase.Err == nil { t.Fatal(err) @@ -252,24 +254,165 @@ func TestStore(t *testing.T) { } } -type nopImageStore struct{} - -func (nopImageStore) Get(ctx context.Context, name string) (images.Image, error) { - return images.Image{}, errdefs.ErrNotFound +func TestLookup(t *testing.T) { + ctx := context.Background() + is := newSimpleImageStore() + for _, name := range []string{ + "registry.io/test1:latest", + "registry.io/test1:v1", + } { + is.Create(ctx, images.Image{ + Name: name, + }) + } + for _, testCase := range []struct { + Name string + ImageStore *Store + Expected []string + Err error + }{ + { + Name: "SingleImage", + ImageStore: &Store{ + imageName: "registry.io/test1:latest", + }, + Expected: []string{"registry.io/test1:latest"}, + }, + { + Name: "MultipleReferences", + ImageStore: &Store{ + imageName: "registry.io/test1:latest", + extraReferences: []Reference{ + { + Name: "registry.io/test1:v1", + }, + }, + }, + Expected: []string{"registry.io/test1:latest", "registry.io/test1:v1"}, + }, + { + Name: "OnlyReferences", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.io/test1:latest", + }, + { + Name: "registry.io/test1:v1", + }, + }, + }, + Expected: []string{"registry.io/test1:latest", "registry.io/test1:v1"}, + }, + { + Name: "UnsupportedPrefix", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.io/test1:latest", + IsPrefix: true, + }, + }, + }, + Err: errdefs.ErrNotImplemented, + }, + } { + t.Run(testCase.Name, func(t *testing.T) { + images, err := testCase.ImageStore.Lookup(ctx, is) + if err != nil { + if !errors.Is(err, testCase.Err) { + t.Errorf("unexpected error %v, expected %v", err, testCase.Err) + } + return + } else if testCase.Err != nil { + t.Fatal("expected error") + } + imageNames := make([]string, len(images)) + for i, img := range images { + imageNames[i] = img.Name + } + sort.Strings(imageNames) + sort.Strings(testCase.Expected) + if len(images) != len(testCase.Expected) { + t.Fatalf("unexpected images:\n\t%v\nexpected:\n\t%v", imageNames, testCase.Expected) + } + for i := range imageNames { + if imageNames[i] != testCase.Expected[i] { + t.Fatalf("unexpected images:\n\t%v\nexpected:\n\t%v", imageNames, testCase.Expected) + } + } + }) + } } -func (nopImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { - return nil, nil +// simpleImageStore is for testing images in memory, +// no filter support +type simpleImageStore struct { + l sync.Mutex + images map[string]images.Image } -func (nopImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { +func newSimpleImageStore() images.Store { + return &simpleImageStore{ + images: make(map[string]images.Image), + } +} + +func (is *simpleImageStore) Get(ctx context.Context, name string) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + img, ok := is.images[name] + if !ok { + return images.Image{}, errdefs.ErrNotFound + } + return img, nil +} + +func (is *simpleImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + var imgs []images.Image + + // filters not supported, return all + for _, img := range is.images { + imgs = append(imgs, img) + } + return imgs, nil +} + +func (is *simpleImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[image.Name]; ok { + return images.Image{}, errdefs.ErrAlreadyExists + } + is.images[image.Name] = image + return image, nil } -func (nopImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { +func (is *simpleImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[image.Name]; !ok { + return images.Image{}, errdefs.ErrNotFound + } + // fieldpaths no supported, update entire image + is.images[image.Name] = image + return image, nil } -func (nopImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { +func (is *simpleImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[name]; !ok { + return errdefs.ErrNotFound + } + delete(is.images, name) + return nil } diff --git a/pkg/transfer/local/export.go b/pkg/transfer/local/export.go index 6c7cff926..069dcbebf 100644 --- a/pkg/transfer/local/export.go +++ b/pkg/transfer/local/export.go @@ -19,10 +19,11 @@ package local import ( "context" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/pkg/transfer" ) -func (ts *localTransferService) exportStream(ctx context.Context, is transfer.ImageExporter, tops *transfer.Config) error { +func (ts *localTransferService) exportStream(ctx context.Context, ig transfer.ImageGetter, is transfer.ImageExporter, tops *transfer.Config) error { ctx, done, err := ts.withLease(ctx) if err != nil { return err @@ -35,7 +36,21 @@ func (ts *localTransferService) exportStream(ctx context.Context, is transfer.Im }) } - err = is.Export(ctx, ts.images, ts.content) + var imgs []images.Image + if il, ok := ig.(transfer.ImageLookup); ok { + imgs, err = il.Lookup(ctx, ts.images) + if err != nil { + return err + } + } else { + img, err := ig.Get(ctx, ts.images) + if err != nil { + return err + } + imgs = append(imgs, img) + } + + err = is.Export(ctx, ts.content, imgs) if err != nil { return err } diff --git a/pkg/transfer/local/transfer.go b/pkg/transfer/local/transfer.go index d0a555d1c..68a1ea7c4 100644 --- a/pkg/transfer/local/transfer.go +++ b/pkg/transfer/local/transfer.go @@ -73,7 +73,7 @@ func (ts *localTransferService) Transfer(ctx context.Context, src interface{}, d case transfer.ImagePusher: return ts.push(ctx, s, d, topts) case transfer.ImageExporter: - return ts.exportStream(ctx, d, topts) + return ts.exportStream(ctx, s, d, topts) } case transfer.ImageImporter: switch d := dest.(type) { diff --git a/pkg/transfer/transfer.go b/pkg/transfer/transfer.go index f87a99061..01df8c3d3 100644 --- a/pkg/transfer/transfer.go +++ b/pkg/transfer/transfer.go @@ -69,9 +69,15 @@ type ImageGetter interface { Get(context.Context, images.Store) (images.Image, error) } +// ImageLookup is a type which returns images from an image store +// based on names or prefixes +type ImageLookup interface { + Lookup(context.Context, images.Store) ([]images.Image, error) +} + // ImageExporter exports images to a writer type ImageExporter interface { - Export(ctx context.Context, is images.Store, cs content.Store) error + Export(context.Context, content.Store, []images.Image) error } // ImageImporter imports an image into a content store