diff --git a/api/next.pb.txt b/api/next.pb.txt index c37a8c9d9..ec7fd3da4 100644 --- a/api/next.pb.txt +++ b/api/next.pb.txt @@ -6857,6 +6857,7 @@ file { file { name: "github.com/containerd/containerd/api/types/transfer/importexport.proto" package: "containerd.types.transfer" + dependency: "github.com/containerd/containerd/api/types/platform.proto" message_type { name: "ImageImportStream" field { @@ -6897,6 +6898,35 @@ file { type: TYPE_STRING json_name: "mediaType" } + field { + name: "platforms" + number: 3 + label: LABEL_REPEATED + type: TYPE_MESSAGE + type_name: ".containerd.types.Platform" + json_name: "platforms" + } + field { + name: "all_platforms" + number: 4 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "allPlatforms" + } + field { + name: "skip_compatibility_manifest" + number: 5 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "skipCompatibilityManifest" + } + field { + name: "skip_non_distributable" + number: 6 + label: LABEL_OPTIONAL + type: TYPE_BOOL + json_name: "skipNonDistributable" + } } options { go_package: "github.com/containerd/containerd/api/types/transfer" diff --git a/api/types/transfer/importexport.pb.go b/api/types/transfer/importexport.pb.go index ca24b3cb3..a2a48ac15 100644 --- a/api/types/transfer/importexport.pb.go +++ b/api/types/transfer/importexport.pb.go @@ -22,6 +22,7 @@ package transfer import ( + types "github.com/containerd/containerd/api/types" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" reflect "reflect" @@ -111,6 +112,14 @@ type ImageExportStream struct { // The binary data is expected to be a raw tar stream. Stream string `protobuf:"bytes,1,opt,name=stream,proto3" json:"stream,omitempty"` MediaType string `protobuf:"bytes,2,opt,name=media_type,json=mediaType,proto3" json:"media_type,omitempty"` + // The specified platforms + Platforms []*types.Platform `protobuf:"bytes,3,rep,name=platforms,proto3" json:"platforms,omitempty"` + // Whether to include all platforms + AllPlatforms bool `protobuf:"varint,4,opt,name=all_platforms,json=allPlatforms,proto3" json:"all_platforms,omitempty"` + // Skips the creation of the Docker compatible manifest.json file + SkipCompatibilityManifest bool `protobuf:"varint,5,opt,name=skip_compatibility_manifest,json=skipCompatibilityManifest,proto3" json:"skip_compatibility_manifest,omitempty"` + // Excludes non-distributable blobs such as Windows base layers. + SkipNonDistributable bool `protobuf:"varint,6,opt,name=skip_non_distributable,json=skipNonDistributable,proto3" json:"skip_non_distributable,omitempty"` } func (x *ImageExportStream) Reset() { @@ -159,6 +168,34 @@ func (x *ImageExportStream) GetMediaType() string { return "" } +func (x *ImageExportStream) GetPlatforms() []*types.Platform { + if x != nil { + return x.Platforms + } + return nil +} + +func (x *ImageExportStream) GetAllPlatforms() bool { + if x != nil { + return x.AllPlatforms + } + return false +} + +func (x *ImageExportStream) GetSkipCompatibilityManifest() bool { + if x != nil { + return x.SkipCompatibilityManifest + } + return false +} + +func (x *ImageExportStream) GetSkipNonDistributable() bool { + if x != nil { + return x.SkipNonDistributable + } + return false +} + var File_github_com_containerd_containerd_api_types_transfer_importexport_proto protoreflect.FileDescriptor var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_rawDesc = []byte{ @@ -168,23 +205,40 @@ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_ 0x6e, 0x73, 0x66, 0x65, 0x72, 0x2f, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x65, 0x78, 0x70, 0x6f, 0x72, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x19, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2e, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x65, 0x72, 0x22, 0x71, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6d, 0x70, 0x6f, - 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x25, 0x0a, 0x0e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x6f, - 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x22, 0x4a, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, - 0x78, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, - 0x70, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, - 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, - 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x66, 0x65, 0x72, 0x1a, 0x39, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, 0x73, 0x2f, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x71, + 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, + 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x6f, + 0x72, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0d, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, + 0x73, 0x22, 0x9f, 0x02, 0x0a, 0x11, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x45, 0x78, 0x70, 0x6f, 0x72, + 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, + 0x1d, 0x0a, 0x0a, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, + 0x0a, 0x09, 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2e, 0x74, + 0x79, 0x70, 0x65, 0x73, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x70, + 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x6c, 0x6c, 0x5f, + 0x70, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x0c, 0x61, 0x6c, 0x6c, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x3e, 0x0a, + 0x1b, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x69, 0x62, 0x69, 0x6c, + 0x69, 0x74, 0x79, 0x5f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x19, 0x73, 0x6b, 0x69, 0x70, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x69, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x79, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, + 0x16, 0x73, 0x6b, 0x69, 0x70, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x64, 0x69, 0x73, 0x74, 0x72, 0x69, + 0x62, 0x75, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, + 0x6b, 0x69, 0x70, 0x4e, 0x6f, 0x6e, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x61, + 0x62, 0x6c, 0x65, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x63, 0x6f, 0x6e, + 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x64, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x74, 0x79, 0x70, 0x65, + 0x73, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x65, 0x72, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x33, } var ( @@ -203,13 +257,15 @@ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_ var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_goTypes = []interface{}{ (*ImageImportStream)(nil), // 0: containerd.types.transfer.ImageImportStream (*ImageExportStream)(nil), // 1: containerd.types.transfer.ImageExportStream + (*types.Platform)(nil), // 2: containerd.types.Platform } var file_github_com_containerd_containerd_api_types_transfer_importexport_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name + 2, // 0: containerd.types.transfer.ImageExportStream.platforms:type_name -> containerd.types.Platform + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name } func init() { file_github_com_containerd_containerd_api_types_transfer_importexport_proto_init() } diff --git a/api/types/transfer/importexport.proto b/api/types/transfer/importexport.proto index 54c9db9c0..c18bae1c6 100644 --- a/api/types/transfer/importexport.proto +++ b/api/types/transfer/importexport.proto @@ -20,6 +20,8 @@ package containerd.types.transfer; option go_package = "github.com/containerd/containerd/api/types/transfer"; +import "github.com/containerd/containerd/api/types/platform.proto"; + message ImageImportStream { // Stream is used to identify the binary input stream for the import operation. // The stream uses the transfer binary stream protocol with the client as the sender. @@ -38,4 +40,13 @@ message ImageExportStream { string stream = 1; string media_type = 2; + + // The specified platforms + repeated types.Platform platforms = 3; + // Whether to include all platforms + bool all_platforms = 4; + // Skips the creation of the Docker compatible manifest.json file + bool skip_compatibility_manifest = 5; + // Excludes non-distributable blobs such as Windows base layers. + bool skip_non_distributable = 6; } diff --git a/cmd/ctr/commands/images/export.go b/cmd/ctr/commands/images/export.go index d4e9be9b2..b61641d86 100644 --- a/cmd/ctr/commands/images/export.go +++ b/cmd/ctr/commands/images/export.go @@ -22,11 +22,15 @@ import ( "io" "os" - "github.com/containerd/containerd/cmd/ctr/commands" - "github.com/containerd/containerd/images/archive" - "github.com/containerd/containerd/platforms" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/urfave/cli" + + "github.com/containerd/containerd/cmd/ctr/commands" + "github.com/containerd/containerd/images/archive" + "github.com/containerd/containerd/pkg/transfer" + tarchive "github.com/containerd/containerd/pkg/transfer/archive" + "github.com/containerd/containerd/pkg/transfer/image" + "github.com/containerd/containerd/platforms" ) var exportCommand = cli.Command{ @@ -58,6 +62,10 @@ When '--all-platforms' is given all images in a manifest list must be available. Name: "all-platforms", Usage: "Exports content from all platforms", }, + cli.BoolTFlag{ + Name: "local", + Usage: "run export locally rather than through transfer API", + }, }, Action: func(context *cli.Context) error { var ( @@ -69,6 +77,61 @@ When '--all-platforms' is given all images in a manifest list must be available. return errors.New("please provide both an output filename and an image reference to export") } + client, ctx, cancel, err := commands.NewClient(context) + if err != nil { + return err + } + defer cancel() + + var w io.WriteCloser + if out == "-" { + w = os.Stdout + } else { + w, err = os.Create(out) + if err != nil { + return err + } + } + defer w.Close() + + if !context.BoolT("local") { + pf, done := ProgressHandler(ctx, os.Stdout) + defer done() + + exportOpts := []tarchive.ExportOpt{} + if pss := context.StringSlice("platform"); len(pss) > 0 { + for _, ps := range pss { + p, err := platforms.Parse(ps) + if err != nil { + return fmt.Errorf("invalid platform %q: %w", ps, err) + } + exportOpts = append(exportOpts, tarchive.WithPlatform(p)) + } + } + if context.Bool("all-platforms") { + exportOpts = append(exportOpts, tarchive.WithAllPlatforms) + } + + if context.Bool("skip-manifest-json") { + exportOpts = append(exportOpts, tarchive.WithSkipCompatibilityManifest) + } + + if context.Bool("skip-non-distributable") { + exportOpts = append(exportOpts, tarchive.WithSkipNonDistributableBlobs) + } + + storeOpts := []image.StoreOpt{} + for _, img := range images { + storeOpts = append(storeOpts, image.WithExtraReference(img)) + } + + return client.Transfer(ctx, + image.NewStore("", storeOpts...), + tarchive.NewImageExportStream(w, "", exportOpts...), + transfer.WithProgress(pf), + ) + } + if pss := context.StringSlice("platform"); len(pss) > 0 { var all []ocispec.Platform for _, ps := range pss { @@ -95,28 +158,11 @@ When '--all-platforms' is given all images in a manifest list must be available. exportOpts = append(exportOpts, archive.WithSkipNonDistributableBlobs()) } - client, ctx, cancel, err := commands.NewClient(context) - if err != nil { - return err - } - defer cancel() - is := client.ImageService() for _, img := range images { exportOpts = append(exportOpts, archive.WithImage(is, img)) } - var w io.WriteCloser - if out == "-" { - w = os.Stdout - } else { - w, err = os.Create(out) - if err != nil { - return err - } - } - defer w.Close() - return client.Export(ctx, w, exportOpts...) }, } diff --git a/images/archive/exporter.go b/images/archive/exporter.go index 40a0a33df..87858a958 100644 --- a/images/archive/exporter.go +++ b/images/archive/exporter.go @@ -89,6 +89,18 @@ func WithImage(is images.Store, name string) ExportOpt { } } +// WithImages adds multiples images to the exported archive. +func WithImages(imgs []images.Image) ExportOpt { + return func(ctx context.Context, o *exportOptions) error { + for _, img := range imgs { + img.Target.Annotations = addNameAnnotation(img.Name, img.Target.Annotations) + o.manifests = append(o.manifests, img.Target) + } + + return nil + } +} + // WithManifest adds a manifest to the exported archive. // When names are given they will be set on the manifest in the // exported archive, creating an index record for each name. diff --git a/pkg/transfer/archive/exporter.go b/pkg/transfer/archive/exporter.go index a5ef20872..80a6d2dc1 100644 --- a/pkg/transfer/archive/exporter.go +++ b/pkg/transfer/archive/exporter.go @@ -20,12 +20,19 @@ import ( "context" "io" + "github.com/containerd/typeurl/v2" + v1 "github.com/opencontainers/image-spec/specs-go/v1" + + "github.com/containerd/containerd/api/types" transfertypes "github.com/containerd/containerd/api/types/transfer" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/archive" "github.com/containerd/containerd/log" "github.com/containerd/containerd/pkg/streaming" "github.com/containerd/containerd/pkg/transfer/plugins" tstreaming "github.com/containerd/containerd/pkg/transfer/streaming" - "github.com/containerd/typeurl/v2" + "github.com/containerd/containerd/platforms" ) func init() { @@ -34,24 +41,74 @@ func init() { plugins.Register(&transfertypes.ImageImportStream{}, &ImageImportStream{}) } -// NewImageExportStream returns a image importer via tar stream -// TODO: Add export options -func NewImageExportStream(stream io.WriteCloser, mediaType string) *ImageExportStream { - return &ImageExportStream{ +type ExportOpt func(*ImageExportStream) + +func WithPlatform(p v1.Platform) ExportOpt { + return func(s *ImageExportStream) { + s.platforms = append(s.platforms, p) + } +} + +func WithAllPlatforms(s *ImageExportStream) { + s.allPlatforms = true +} + +func WithSkipCompatibilityManifest(s *ImageExportStream) { + s.skipCompatibilityManifest = true +} + +func WithSkipNonDistributableBlobs(s *ImageExportStream) { + s.skipNonDistributable = true +} + +// NewImageExportStream returns an image exporter via tar stream +func NewImageExportStream(stream io.WriteCloser, mediaType string, opts ...ExportOpt) *ImageExportStream { + s := &ImageExportStream{ stream: stream, mediaType: mediaType, } + for _, opt := range opts { + opt(s) + } + return s } type ImageExportStream struct { stream io.WriteCloser mediaType string + + platforms []v1.Platform + allPlatforms bool + skipCompatibilityManifest bool + skipNonDistributable bool } func (iis *ImageExportStream) ExportStream(context.Context) (io.WriteCloser, string, error) { return iis.stream, iis.mediaType, nil } +func (iis *ImageExportStream) Export(ctx context.Context, cs content.Store, imgs []images.Image) error { + opts := []archive.ExportOpt{ + archive.WithImages(imgs), + } + + if len(iis.platforms) > 0 { + opts = append(opts, archive.WithPlatform(platforms.Ordered(iis.platforms...))) + } else { + opts = append(opts, archive.WithPlatform(platforms.DefaultStrict())) + } + if iis.allPlatforms { + opts = append(opts, archive.WithAllPlatforms()) + } + if iis.skipCompatibilityManifest { + opts = append(opts, archive.WithSkipDockerManifest()) + } + if iis.skipNonDistributable { + opts = append(opts, archive.WithSkipNonDistributableBlobs()) + } + return archive.Export(ctx, cs, iis.stream, opts...) +} + func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) { sid := tstreaming.GenerateID("export") stream, err := sm.Create(ctx, sid) @@ -67,9 +124,21 @@ func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.Strea iis.stream.Close() }() + var specified []*types.Platform + for _, p := range iis.platforms { + specified = append(specified, &types.Platform{ + OS: p.OS, + Architecture: p.Architecture, + Variant: p.Variant, + }) + } s := &transfertypes.ImageExportStream{ - Stream: sid, - MediaType: iis.mediaType, + Stream: sid, + MediaType: iis.mediaType, + Platforms: specified, + AllPlatforms: iis.allPlatforms, + SkipCompatibilityManifest: iis.skipCompatibilityManifest, + SkipNonDistributable: iis.skipNonDistributable, } return typeurl.MarshalAny(s) @@ -87,8 +156,21 @@ func (iis *ImageExportStream) UnmarshalAny(ctx context.Context, sm streaming.Str return err } + var specified []v1.Platform + for _, p := range s.Platforms { + specified = append(specified, v1.Platform{ + OS: p.OS, + Architecture: p.Architecture, + Variant: p.Variant, + }) + } + iis.stream = tstreaming.WriteByteStream(ctx, stream) iis.mediaType = s.MediaType + iis.platforms = specified + iis.allPlatforms = s.AllPlatforms + iis.skipCompatibilityManifest = s.SkipCompatibilityManifest + iis.skipNonDistributable = s.SkipNonDistributable return nil } diff --git a/pkg/transfer/image/imagestore.go b/pkg/transfer/image/imagestore.go index 547228a15..b26f25605 100644 --- a/pkg/transfer/image/imagestore.go +++ b/pkg/transfer/image/imagestore.go @@ -325,6 +325,28 @@ func (is *Store) Get(ctx context.Context, store images.Store) (images.Image, err return store.Get(ctx, is.imageName) } +func (is *Store) Lookup(ctx context.Context, store images.Store) ([]images.Image, error) { + var imgs []images.Image + if is.imageName != "" { + img, err := store.Get(ctx, is.imageName) + if err != nil { + return nil, err + } + imgs = append(imgs, img) + } + for _, ref := range is.extraReferences { + if ref.IsPrefix { + return nil, fmt.Errorf("prefix lookup on export not implemented: %w", errdefs.ErrNotImplemented) + } + img, err := store.Get(ctx, ref.Name) + if err != nil { + return nil, err + } + imgs = append(imgs, img) + } + return imgs, nil +} + func (is *Store) UnpackPlatforms() []transfer.UnpackConfiguration { unpacks := make([]transfer.UnpackConfiguration, len(is.unpacks)) for i, uc := range is.unpacks { diff --git a/pkg/transfer/image/imagestore_test.go b/pkg/transfer/image/imagestore_test.go index fbca0e2c3..cefa0c1ef 100644 --- a/pkg/transfer/image/imagestore_test.go +++ b/pkg/transfer/image/imagestore_test.go @@ -19,6 +19,8 @@ package image import ( "context" "errors" + "sort" + "sync" "testing" "github.com/containerd/containerd/errdefs" @@ -222,7 +224,7 @@ func TestStore(t *testing.T) { desc.Annotations["io.containerd.import.ref-source"] = "annotation" } t.Run(name, func(t *testing.T) { - imgs, err := testCase.ImageStore.Store(context.Background(), desc, nopImageStore{}) + imgs, err := testCase.ImageStore.Store(context.Background(), desc, newSimpleImageStore()) if err != nil { if testCase.Err == nil { t.Fatal(err) @@ -252,24 +254,165 @@ func TestStore(t *testing.T) { } } -type nopImageStore struct{} - -func (nopImageStore) Get(ctx context.Context, name string) (images.Image, error) { - return images.Image{}, errdefs.ErrNotFound +func TestLookup(t *testing.T) { + ctx := context.Background() + is := newSimpleImageStore() + for _, name := range []string{ + "registry.io/test1:latest", + "registry.io/test1:v1", + } { + is.Create(ctx, images.Image{ + Name: name, + }) + } + for _, testCase := range []struct { + Name string + ImageStore *Store + Expected []string + Err error + }{ + { + Name: "SingleImage", + ImageStore: &Store{ + imageName: "registry.io/test1:latest", + }, + Expected: []string{"registry.io/test1:latest"}, + }, + { + Name: "MultipleReferences", + ImageStore: &Store{ + imageName: "registry.io/test1:latest", + extraReferences: []Reference{ + { + Name: "registry.io/test1:v1", + }, + }, + }, + Expected: []string{"registry.io/test1:latest", "registry.io/test1:v1"}, + }, + { + Name: "OnlyReferences", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.io/test1:latest", + }, + { + Name: "registry.io/test1:v1", + }, + }, + }, + Expected: []string{"registry.io/test1:latest", "registry.io/test1:v1"}, + }, + { + Name: "UnsupportedPrefix", + ImageStore: &Store{ + extraReferences: []Reference{ + { + Name: "registry.io/test1:latest", + IsPrefix: true, + }, + }, + }, + Err: errdefs.ErrNotImplemented, + }, + } { + t.Run(testCase.Name, func(t *testing.T) { + images, err := testCase.ImageStore.Lookup(ctx, is) + if err != nil { + if !errors.Is(err, testCase.Err) { + t.Errorf("unexpected error %v, expected %v", err, testCase.Err) + } + return + } else if testCase.Err != nil { + t.Fatal("expected error") + } + imageNames := make([]string, len(images)) + for i, img := range images { + imageNames[i] = img.Name + } + sort.Strings(imageNames) + sort.Strings(testCase.Expected) + if len(images) != len(testCase.Expected) { + t.Fatalf("unexpected images:\n\t%v\nexpected:\n\t%v", imageNames, testCase.Expected) + } + for i := range imageNames { + if imageNames[i] != testCase.Expected[i] { + t.Fatalf("unexpected images:\n\t%v\nexpected:\n\t%v", imageNames, testCase.Expected) + } + } + }) + } } -func (nopImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { - return nil, nil +// simpleImageStore is for testing images in memory, +// no filter support +type simpleImageStore struct { + l sync.Mutex + images map[string]images.Image } -func (nopImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { +func newSimpleImageStore() images.Store { + return &simpleImageStore{ + images: make(map[string]images.Image), + } +} + +func (is *simpleImageStore) Get(ctx context.Context, name string) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + img, ok := is.images[name] + if !ok { + return images.Image{}, errdefs.ErrNotFound + } + return img, nil +} + +func (is *simpleImageStore) List(ctx context.Context, filters ...string) ([]images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + var imgs []images.Image + + // filters not supported, return all + for _, img := range is.images { + imgs = append(imgs, img) + } + return imgs, nil +} + +func (is *simpleImageStore) Create(ctx context.Context, image images.Image) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[image.Name]; ok { + return images.Image{}, errdefs.ErrAlreadyExists + } + is.images[image.Name] = image + return image, nil } -func (nopImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { +func (is *simpleImageStore) Update(ctx context.Context, image images.Image, fieldpaths ...string) (images.Image, error) { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[image.Name]; !ok { + return images.Image{}, errdefs.ErrNotFound + } + // fieldpaths no supported, update entire image + is.images[image.Name] = image + return image, nil } -func (nopImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { +func (is *simpleImageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + is.l.Lock() + defer is.l.Unlock() + + if _, ok := is.images[name]; !ok { + return errdefs.ErrNotFound + } + delete(is.images, name) + return nil } diff --git a/pkg/transfer/local/export.go b/pkg/transfer/local/export.go new file mode 100644 index 000000000..069dcbebf --- /dev/null +++ b/pkg/transfer/local/export.go @@ -0,0 +1,64 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package local + +import ( + "context" + + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/pkg/transfer" +) + +func (ts *localTransferService) exportStream(ctx context.Context, ig transfer.ImageGetter, is transfer.ImageExporter, tops *transfer.Config) error { + ctx, done, err := ts.withLease(ctx) + if err != nil { + return err + } + defer done(ctx) + + if tops.Progress != nil { + tops.Progress(transfer.Progress{ + Event: "Exporting", + }) + } + + var imgs []images.Image + if il, ok := ig.(transfer.ImageLookup); ok { + imgs, err = il.Lookup(ctx, ts.images) + if err != nil { + return err + } + } else { + img, err := ig.Get(ctx, ts.images) + if err != nil { + return err + } + imgs = append(imgs, img) + } + + err = is.Export(ctx, ts.content, imgs) + if err != nil { + return err + } + + if tops.Progress != nil { + tops.Progress(transfer.Progress{ + Event: "Completed export", + }) + } + return nil +} diff --git a/pkg/transfer/local/transfer.go b/pkg/transfer/local/transfer.go index 0189290dd..68a1ea7c4 100644 --- a/pkg/transfer/local/transfer.go +++ b/pkg/transfer/local/transfer.go @@ -72,6 +72,8 @@ func (ts *localTransferService) Transfer(ctx context.Context, src interface{}, d switch d := dest.(type) { case transfer.ImagePusher: return ts.push(ctx, s, d, topts) + case transfer.ImageExporter: + return ts.exportStream(ctx, s, d, topts) } case transfer.ImageImporter: switch d := dest.(type) { diff --git a/pkg/transfer/transfer.go b/pkg/transfer/transfer.go index a9c4fcd73..01df8c3d3 100644 --- a/pkg/transfer/transfer.go +++ b/pkg/transfer/transfer.go @@ -69,6 +69,17 @@ type ImageGetter interface { Get(context.Context, images.Store) (images.Image, error) } +// ImageLookup is a type which returns images from an image store +// based on names or prefixes +type ImageLookup interface { + Lookup(context.Context, images.Store) ([]images.Image, error) +} + +// ImageExporter exports images to a writer +type ImageExporter interface { + Export(context.Context, content.Store, []images.Image) error +} + // ImageImporter imports an image into a content store type ImageImporter interface { Import(context.Context, content.Store) (ocispec.Descriptor, error)