feat: export images using Transfer api

Signed-off-by: Jian Zeng <anonymousknight96@gmail.com>

cmd/ctr/commands/images/export.go

@@ -22,11 +22,15 @@ import (
 	"io"
 	"os"
 
-	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/images/archive"
-	"github.com/containerd/containerd/platforms"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/urfave/cli"
+
+	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/containerd/images/archive"
+	"github.com/containerd/containerd/pkg/transfer"
+	tarchive "github.com/containerd/containerd/pkg/transfer/archive"
+	"github.com/containerd/containerd/pkg/transfer/image"
+	"github.com/containerd/containerd/platforms"
 )
 
 var exportCommand = cli.Command{
@@ -58,6 +62,10 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			Name:  "all-platforms",
 			Usage: "Exports content from all platforms",
 		},
+		cli.BoolTFlag{
+			Name:  "local",
+			Usage: "run export locally rather than through transfer API",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		var (
@@ -69,6 +77,56 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			return errors.New("please provide both an output filename and an image reference to export")
 		}
 
+		client, ctx, cancel, err := commands.NewClient(context)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		var w io.WriteCloser
+		if out == "-" {
+			w = os.Stdout
+		} else {
+			w, err = os.Create(out)
+			if err != nil {
+				return err
+			}
+		}
+		defer w.Close()
+
+		if !context.BoolT("local") {
+			pf, done := ProgressHandler(ctx, os.Stdout)
+			defer done()
+
+			var specified []ocispec.Platform
+			if pss := context.StringSlice("platform"); len(pss) > 0 {
+				for _, ps := range pss {
+					p, err := platforms.Parse(ps)
+					if err != nil {
+						return fmt.Errorf("invalid platform %q: %w", ps, err)
+					}
+					specified = append(specified, p)
+				}
+			}
+
+			err := client.Transfer(ctx,
+				image.NewStore(""), // a dummy image store
+				tarchive.NewImageExportStream(w, "", tarchive.ExportOptions{
+					Images:               images,
+					Platforms:            specified,
+					AllPlatforms:         context.Bool("all-platforms"),
+					SkipNonDistributable: context.Bool("skip-non-distributable"),
+					SkipDockerManifest:   context.Bool("skip-manifest-json"),
+				}),
+				transfer.WithProgress(pf),
+			)
+			if err != nil {
+				return err
+			}
+
+			return nil
+		}
+
 		if pss := context.StringSlice("platform"); len(pss) > 0 {
 			var all []ocispec.Platform
 			for _, ps := range pss {
@@ -95,28 +153,11 @@ When '--all-platforms' is given all images in a manifest list must be available.
 			exportOpts = append(exportOpts, archive.WithSkipNonDistributableBlobs())
 		}
 
-		client, ctx, cancel, err := commands.NewClient(context)
-		if err != nil {
-			return err
-		}
-		defer cancel()
-
 		is := client.ImageService()
 		for _, img := range images {
 			exportOpts = append(exportOpts, archive.WithImage(is, img))
 		}
 
-		var w io.WriteCloser
-		if out == "-" {
-			w = os.Stdout
-		} else {
-			w, err = os.Create(out)
-			if err != nil {
-				return err
-			}
-		}
-		defer w.Close()
-
 		return client.Export(ctx, w, exportOpts...)
 	},
 }

pkg/transfer/archive/exporter.go

@@ -20,12 +20,19 @@ import (
 	"context"
 	"io"
 
+	"github.com/containerd/typeurl/v2"
+	v1 "github.com/opencontainers/image-spec/specs-go/v1"
+
+	"github.com/containerd/containerd/api/types"
 	transfertypes "github.com/containerd/containerd/api/types/transfer"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/images/archive"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/pkg/streaming"
 	"github.com/containerd/containerd/pkg/transfer/plugins"
 	tstreaming "github.com/containerd/containerd/pkg/transfer/streaming"
-	"github.com/containerd/typeurl/v2"
+	"github.com/containerd/containerd/platforms"
 )
 
 func init() {
@@ -34,24 +41,65 @@ func init() {
 	plugins.Register(&transfertypes.ImageImportStream{}, &ImageImportStream{})
 }
 
-// NewImageExportStream returns a image importer via tar stream
+type ExportOptions struct {
-// TODO: Add export options
+	Images               []string
-func NewImageExportStream(stream io.WriteCloser, mediaType string) *ImageExportStream {
+	Platforms            []v1.Platform
+	AllPlatforms         bool
+	SkipDockerManifest   bool
+	SkipNonDistributable bool
+}
+
+// NewImageExportStream returns an image exporter via tar stream
+func NewImageExportStream(stream io.WriteCloser, mediaType string, opts ExportOptions) *ImageExportStream {
 	return &ImageExportStream{
 		stream:    stream,
 		mediaType: mediaType,
+
+		images:               opts.Images,
+		platforms:            opts.Platforms,
+		allPlatforms:         opts.AllPlatforms,
+		skipDockerManifest:   opts.SkipDockerManifest,
+		skipNonDistributable: opts.SkipNonDistributable,
 	}
 }
 
 type ImageExportStream struct {
 	stream    io.WriteCloser
 	mediaType string
+
+	images               []string
+	platforms            []v1.Platform
+	allPlatforms         bool
+	skipDockerManifest   bool
+	skipNonDistributable bool
 }
 
 func (iis *ImageExportStream) ExportStream(context.Context) (io.WriteCloser, string, error) {
 	return iis.stream, iis.mediaType, nil
 }
 
+func (iis *ImageExportStream) Export(ctx context.Context, is images.Store, cs content.Store) error {
+	var opts []archive.ExportOpt
+	for _, img := range iis.images {
+		opts = append(opts, archive.WithImage(is, img))
+	}
+	if len(iis.platforms) > 0 {
+		opts = append(opts, archive.WithPlatform(platforms.Ordered(iis.platforms...)))
+	} else {
+		opts = append(opts, archive.WithPlatform(platforms.DefaultStrict()))
+	}
+	if iis.allPlatforms {
+		opts = append(opts, archive.WithAllPlatforms())
+	}
+	if iis.skipDockerManifest {
+		opts = append(opts, archive.WithSkipDockerManifest())
+	}
+	if iis.skipNonDistributable {
+		opts = append(opts, archive.WithSkipNonDistributableBlobs())
+	}
+	return archive.Export(ctx, cs, iis.stream, opts...)
+}
+
 func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.StreamCreator) (typeurl.Any, error) {
 	sid := tstreaming.GenerateID("export")
 	stream, err := sm.Create(ctx, sid)
@@ -67,9 +115,22 @@ func (iis *ImageExportStream) MarshalAny(ctx context.Context, sm streaming.Strea
 		iis.stream.Close()
 	}()
 
+	var specified []*types.Platform
+	for _, p := range iis.platforms {
+		specified = append(specified, &types.Platform{
+			OS:           p.OS,
+			Architecture: p.Architecture,
+			Variant:      p.Variant,
+		})
+	}
 	s := &transfertypes.ImageExportStream{
-		Stream:    sid,
+		Stream:               sid,
-		MediaType: iis.mediaType,
+		MediaType:            iis.mediaType,
+		Images:               iis.images,
+		Platforms:            specified,
+		AllPlatforms:         iis.allPlatforms,
+		SkipDockerManifest:   iis.skipDockerManifest,
+		SkipNonDistributable: iis.skipNonDistributable,
 	}
 
 	return typeurl.MarshalAny(s)
@@ -87,8 +148,22 @@ func (iis *ImageExportStream) UnmarshalAny(ctx context.Context, sm streaming.Str
 		return err
 	}
 
+	var specified []v1.Platform
+	for _, p := range s.Platforms {
+		specified = append(specified, v1.Platform{
+			OS:           p.OS,
+			Architecture: p.Architecture,
+			Variant:      p.Variant,
+		})
+	}
+
 	iis.stream = tstreaming.WriteByteStream(ctx, stream)
 	iis.mediaType = s.MediaType
+	iis.images = s.Images
+	iis.platforms = specified
+	iis.allPlatforms = s.AllPlatforms
+	iis.skipDockerManifest = s.SkipDockerManifest
+	iis.skipNonDistributable = s.SkipNonDistributable
 
 	return nil
 }

pkg/transfer/local/export.go

@@ -0,0 +1,49 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package local
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/pkg/transfer"
+)
+
+func (ts *localTransferService) exportStream(ctx context.Context, is transfer.ImageExporter, tops *transfer.Config) error {
+	ctx, done, err := ts.withLease(ctx)
+	if err != nil {
+		return err
+	}
+	defer done(ctx)
+
+	if tops.Progress != nil {
+		tops.Progress(transfer.Progress{
+			Event: "Exporting",
+		})
+	}
+
+	err = is.Export(ctx, ts.images, ts.content)
+	if err != nil {
+		return err
+	}
+
+	if tops.Progress != nil {
+		tops.Progress(transfer.Progress{
+			Event: "Completed export",
+		})
+	}
+	return nil
+}

pkg/transfer/local/transfer.go

@@ -72,6 +72,8 @@ func (ts *localTransferService) Transfer(ctx context.Context, src interface{}, d
 		switch d := dest.(type) {
 		case transfer.ImagePusher:
 			return ts.push(ctx, s, d, topts)
+		case transfer.ImageExporter:
+			return ts.exportStream(ctx, d, topts)
 		}
 	case transfer.ImageImporter:
 		switch d := dest.(type) {

pkg/transfer/transfer.go

@@ -69,6 +69,11 @@ type ImageGetter interface {
 	Get(context.Context, images.Store) (images.Image, error)
 }
 
+// ImageExporter exports images to a writer
+type ImageExporter interface {
+	Export(ctx context.Context, is images.Store, cs content.Store) error
+}
+
 // ImageImporter imports an image into a content store
 type ImageImporter interface {
 	Import(context.Context, content.Store) (ocispec.Descriptor, error)