Update image export to support Docker format

Add manifest.json file which is used by Docker
to import images.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

cmd/ctr/commands/images/export.go

@@ -21,9 +21,8 @@ import (
 	"os"
 
 	"github.com/containerd/containerd/cmd/ctr/commands"
-	"github.com/containerd/containerd/images/oci"
+	"github.com/containerd/containerd/images/archive"
-	"github.com/containerd/containerd/reference"
+	"github.com/containerd/containerd/platforms"
-	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
@@ -31,26 +30,24 @@ import (
 
 var exportCommand = cli.Command{
 	Name:      "export",
-	Usage:     "export an image",
+	Usage:     "export images",
-	ArgsUsage: "[flags] <out> <image>",
+	ArgsUsage: "[flags] <out> <image> ...",
-	Description: `Export an image to a tar stream.
+	Description: `Export images to an OCI tar archive.
-Currently, only OCI format is supported.
+
+Tar output is formatted as an OCI archive, a Docker manifest is provided for the platform.
+Use '--skip-manifest-json' to avoid including the Docker manifest.json file.
+Use '--platform' to define the output platform.
+When '--all-platforms' is given all images in a manifest list must be available.
 `,
 	Flags: []cli.Flag{
-		// TODO(AkihiroSuda): make this map[string]string as in moby/moby#33355?
+		cli.BoolFlag{
-		cli.StringFlag{
+			Name:  "skip-manifest-json",
-			Name:  "oci-ref-name",
+			Usage: "do not add Docker compatible manifest.json to archive",
-			Value: "",
-			Usage: "override org.opencontainers.image.ref.name annotation",
 		},
-		cli.StringFlag{
+		cli.StringSliceFlag{
-			Name:  "manifest",
+			Name:  "platform",
-			Usage: "digest of manifest",
+			Usage: "Pull content from a specific platform",
-		},
+			Value: &cli.StringSlice{},
-		cli.StringFlag{
-			Name:  "manifest-type",
-			Usage: "media type of manifest digest",
-			Value: ocispec.MediaTypeImageManifest,
 		},
 		cli.BoolFlag{
 			Name:  "all-platforms",
@@ -59,43 +56,47 @@ Currently, only OCI format is supported.
 	},
 	Action: func(context *cli.Context) error {
 		var (
-			out   = context.Args().First()
+			out        = context.Args().First()
-			local = context.Args().Get(1)
+			images     = context.Args().Tail()
-			desc  ocispec.Descriptor
+			exportOpts = []archive.ExportOpt{}
 		)
-		if out == "" || local == "" {
+		if out == "" || len(images) == 0 {
 			return errors.New("please provide both an output filename and an image reference to export")
 		}
+
+		if pss := context.StringSlice("platform"); len(pss) > 0 {
+			var all []ocispec.Platform
+			for _, ps := range pss {
+				p, err := platforms.Parse(ps)
+				if err != nil {
+					return errors.Wrapf(err, "invalid platform %q", ps)
+				}
+				all = append(all, p)
+			}
+			exportOpts = append(exportOpts, archive.WithPlatform(platforms.Ordered(all...)))
+		} else {
+			exportOpts = append(exportOpts, archive.WithPlatform(platforms.Default()))
+		}
+
+		if context.Bool("all-platforms") {
+			exportOpts = append(exportOpts, archive.WithAllPlatforms())
+		}
+
+		if context.Bool("skip-manifest-json") {
+			exportOpts = append(exportOpts, archive.WithSkipDockerManifest())
+		}
+
 		client, ctx, cancel, err := commands.NewClient(context)
 		if err != nil {
 			return err
 		}
 		defer cancel()
-		if manifest := context.String("manifest"); manifest != "" {
+
-			desc.Digest, err = digest.Parse(manifest)
+		is := client.ImageService()
-			if err != nil {
+		for _, img := range images {
-				return errors.Wrap(err, "invalid manifest digest")
+			exportOpts = append(exportOpts, archive.WithImage(is, img))
-			}
-			desc.MediaType = context.String("manifest-type")
-		} else {
-			img, err := client.ImageService().Get(ctx, local)
-			if err != nil {
-				return errors.Wrap(err, "unable to resolve image to manifest")
-			}
-			desc = img.Target
 		}
 
-		if desc.Annotations == nil {
-			desc.Annotations = make(map[string]string)
-		}
-		if s, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok || s == "" {
-			if ociRefName := determineOCIRefName(local); ociRefName != "" {
-				desc.Annotations[ocispec.AnnotationRefName] = ociRefName
-			}
-			if ociRefName := context.String("oci-ref-name"); ociRefName != "" {
-				desc.Annotations[ocispec.AnnotationRefName] = ociRefName
-			}
-		}
 		var w io.WriteCloser
 		if out == "-" {
 			w = os.Stdout
@@ -105,32 +106,8 @@ Currently, only OCI format is supported.
 				return nil
 			}
 		}
+		defer w.Close()
 
-		var (
+		return client.Export(ctx, w, exportOpts...)
-			exportOpts []oci.V1ExporterOpt
-		)
-
-		exportOpts = append(exportOpts, oci.WithAllPlatforms(context.Bool("all-platforms")))
-
-		r, err := client.Export(ctx, desc, exportOpts...)
-		if err != nil {
-			return err
-		}
-		if _, err := io.Copy(w, r); err != nil {
-			return err
-		}
-		if err := w.Close(); err != nil {
-			return err
-		}
-		return r.Close()
 	},
 }
-
-func determineOCIRefName(local string) string {
-	refspec, err := reference.Parse(local)
-	if err != nil {
-		return ""
-	}
-	tag, _ := reference.SplitObject(refspec.Object)
-	return tag
-}

export.go

@@ -20,26 +20,12 @@ import (
 	"context"
 	"io"
 
-	"github.com/containerd/containerd/images/oci"
+	"github.com/containerd/containerd/images/archive"
-
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 )
 
-// Export exports an image to a Tar stream.
+// Export exports images to a Tar stream.
-// OCI format is used by default.
+// The tar archive is in OCI format with a Docker compatible manifest
-// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
+// when a single target platform is given.
-// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream.
+func (c *Client) Export(ctx context.Context, w io.Writer, opts ...archive.ExportOpt) error {
-func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...oci.V1ExporterOpt) (io.ReadCloser, error) {
+	return archive.Export(ctx, c.ContentStore(), w, opts...)
-
-	exporter, err := oci.ResolveV1ExportOpt(opts...)
-	if err != nil {
-		return nil, err
-	}
-
-	pr, pw := io.Pipe()
-	go func() {
-		pw.CloseWithError(errors.Wrap(exporter.Export(ctx, c.ContentStore(), desc, pw), "export failed"))
-	}()
-	return pr, nil
 }

export_test.go

@@ -18,13 +18,17 @@ package containerd
 
 import (
 	"archive/tar"
+	"bytes"
 	"io"
 	"runtime"
 	"testing"
+
+	"github.com/containerd/containerd/images/archive"
+	"github.com/containerd/containerd/platforms"
 )
 
-// TestOCIExport exports testImage as a tar stream
+// TestExport exports testImage as a tar stream
-func TestOCIExport(t *testing.T) {
+func TestExport(t *testing.T) {
 	// TODO: support windows
 	if testing.Short() || runtime.GOOS == "windows" {
 		t.Skip()
@@ -38,15 +42,16 @@ func TestOCIExport(t *testing.T) {
 	}
 	defer client.Close()
 
-	pulled, err := client.Fetch(ctx, testImage)
+	_, err = client.Fetch(ctx, testImage)
 	if err != nil {
 		t.Fatal(err)
 	}
-	exportedStream, err := client.Export(ctx, pulled.Target)
+	wb := bytes.NewBuffer(nil)
+	err = client.Export(ctx, wb, archive.WithPlatform(platforms.Default()), archive.WithImage(client.ImageService(), testImage))
 	if err != nil {
 		t.Fatal(err)
 	}
-	assertOCITar(t, exportedStream)
+	assertOCITar(t, bytes.NewReader(wb.Bytes()))
 }
 
 func assertOCITar(t *testing.T, r io.Reader) {

images/annotations.go

@@ -0,0 +1,28 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package images
+
+const (
+	// AnnotationImageName is an annotation on a Descriptor in an index.json
+	// containing the `Name` value as used by an `Image` struct
+	AnnotationImageName = "io.containerd.image.name"
+
+	// AnnotationImageNamePrefix is used the same way as AnnotationImageName
+	// but may be used to refer to additional names in the annotation map
+	// using user-defined suffixes (i.e. "extra.1")
+	AnnotationImageNamePrefix = AnnotationImageName + "."
+)

images/archive/exporter.go

@@ -0,0 +1,490 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package archive
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	digest "github.com/opencontainers/go-digest"
+	ocispecs "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+type exportOptions struct {
+	manifests          []ocispec.Descriptor
+	platform           platforms.MatchComparer
+	allPlatforms       bool
+	skipDockerManifest bool
+}
+
+// ExportOpt defines options for configuring exported descriptors
+type ExportOpt func(context.Context, *exportOptions) error
+
+// WithPlatform defines the platform to require manifest lists have
+// not exporting all platforms.
+// Additionally, platform is used to resolve image configs for
+// Docker v1.1, v1.2 format compatibility.
+func WithPlatform(p platforms.MatchComparer) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.platform = p
+		return nil
+	}
+}
+
+// WithAllPlatforms exports all manifests from a manifest list.
+// Missing content will fail the export.
+func WithAllPlatforms() ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.allPlatforms = true
+		return nil
+	}
+}
+
+// WithSkipDockerManifest skips creation of the Docker compatible
+// manifest.json file.
+func WithSkipDockerManifest() ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		o.skipDockerManifest = true
+		return nil
+	}
+}
+
+// WithImage adds the provided images to the exported archive.
+func WithImage(is images.Store, name string) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		img, err := is.Get(ctx, name)
+		if err != nil {
+			return err
+		}
+
+		var i int
+		o.manifests, i = appendDescriptor(o.manifests, img.Target)
+		o.manifests[i].Annotations = addNameAnnotation(name, o.manifests[i].Annotations)
+
+		return nil
+	}
+}
+
+// WithManifest adds a manifest to the exported archive.
+// It is up to caller to put name annotation to on the manifest
+// descriptor if needed.
+func WithManifest(manifest ocispec.Descriptor) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		var i int
+		o.manifests, i = appendDescriptor(o.manifests, manifest)
+		o.manifests[i].Annotations = manifest.Annotations
+		return nil
+	}
+}
+
+// WithNamedManifest adds a manifest to the exported archive
+// with the provided names.
+func WithNamedManifest(manifest ocispec.Descriptor, names ...string) ExportOpt {
+	return func(ctx context.Context, o *exportOptions) error {
+		var i int
+		o.manifests, i = appendDescriptor(o.manifests, manifest)
+		for _, name := range names {
+			o.manifests[i].Annotations = addNameAnnotation(name, o.manifests[i].Annotations)
+		}
+
+		return nil
+	}
+}
+
+func appendDescriptor(descs []ocispec.Descriptor, desc ocispec.Descriptor) ([]ocispec.Descriptor, int) {
+	i := 0
+	for i < len(descs) {
+		if descs[i].Digest == desc.Digest {
+			return descs, i
+		}
+		i++
+	}
+	return append(descs, desc), i
+}
+
+func addNameAnnotation(name string, annotations map[string]string) map[string]string {
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+
+	i := 0
+	for {
+		key := images.AnnotationImageName
+		if i > 0 {
+			key = fmt.Sprintf("%sextra.%d", images.AnnotationImageNamePrefix, i)
+		}
+		i++
+
+		if val, ok := annotations[key]; ok {
+			if val != name {
+				continue
+			}
+		} else {
+			annotations[key] = name
+		}
+		break
+	}
+	return annotations
+}
+
+// Export implements Exporter.
+func Export(ctx context.Context, store content.Provider, writer io.Writer, opts ...ExportOpt) error {
+	var eo exportOptions
+	for _, opt := range opts {
+		if err := opt(ctx, &eo); err != nil {
+			return err
+		}
+	}
+
+	records := []tarRecord{
+		ociLayoutFile(""),
+		ociIndexRecord(eo.manifests),
+	}
+
+	algorithms := map[string]struct{}{}
+	manifestTags := map[string]ocispec.Descriptor{}
+	for _, desc := range eo.manifests {
+		switch desc.MediaType {
+		case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+			r, err := getRecords(ctx, store, desc, algorithms)
+			if err != nil {
+				return err
+			}
+			records = append(records, r...)
+
+			for _, name := range imageNames(desc.Annotations) {
+				manifestTags[name] = desc
+			}
+		case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+			records = append(records, blobRecord(store, desc))
+
+			p, err := content.ReadBlob(ctx, store, desc)
+			if err != nil {
+				return err
+			}
+
+			var index ocispec.Index
+			if err := json.Unmarshal(p, &index); err != nil {
+				return err
+			}
+
+			names := imageNames(desc.Annotations)
+			var manifests []ocispec.Descriptor
+			for _, m := range index.Manifests {
+				if eo.platform != nil {
+					if m.Platform == nil || eo.platform.Match(*m.Platform) {
+						manifests = append(manifests, m)
+					} else if !eo.allPlatforms {
+						continue
+					}
+				}
+
+				r, err := getRecords(ctx, store, m, algorithms)
+				if err != nil {
+					return err
+				}
+
+				records = append(records, r...)
+			}
+
+			if len(names) > 0 && !eo.skipDockerManifest {
+				if len(manifests) >= 1 {
+					if len(manifests) > 1 {
+						sort.SliceStable(manifests, func(i, j int) bool {
+							if manifests[i].Platform == nil {
+								return false
+							}
+							if manifests[j].Platform == nil {
+								return true
+							}
+							return eo.platform.Less(*manifests[i].Platform, *manifests[j].Platform)
+						})
+					}
+					for _, name := range names {
+						manifestTags[name] = manifests[0]
+					}
+				} else if eo.platform != nil {
+					return errors.Wrap(errdefs.ErrNotFound, "no manifest found for platform")
+				}
+			}
+		default:
+			return errors.Wrap(errdefs.ErrInvalidArgument, "only manifests may be exported")
+		}
+	}
+
+	if len(manifestTags) > 0 {
+		tr, err := manifestsRecord(ctx, store, manifestTags)
+		if err != nil {
+			return errors.Wrap(err, "unable to create manifests file")
+		}
+
+		records = append(records, tr)
+	}
+
+	if len(algorithms) > 0 {
+		records = append(records, directoryRecord("blobs/", 0755))
+		for alg := range algorithms {
+			records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
+		}
+	}
+
+	tw := tar.NewWriter(writer)
+	defer tw.Close()
+	return writeTar(ctx, tw, records)
+}
+
+func imageNames(annotations map[string]string) []string {
+	var names []string
+	for k, v := range annotations {
+		if k == images.AnnotationImageName || strings.HasPrefix(k, images.AnnotationImageName) {
+			names = append(names, v)
+		}
+	}
+	return names
+}
+
+func getRecords(ctx context.Context, store content.Provider, desc ocispec.Descriptor, algorithms map[string]struct{}) ([]tarRecord, error) {
+	var records []tarRecord
+	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		records = append(records, blobRecord(store, desc))
+		algorithms[desc.Digest.Algorithm().String()] = struct{}{}
+		return nil, nil
+	}
+
+	childrenHandler := images.ChildrenHandler(store)
+
+	handlers := images.Handlers(
+		childrenHandler,
+		images.HandlerFunc(exportHandler),
+	)
+
+	// Walk sequentially since the number of fetchs is likely one and doing in
+	// parallel requires locking the export handler
+	if err := images.Walk(ctx, handlers, desc); err != nil {
+		return nil, err
+	}
+
+	return records, nil
+}
+
+type tarRecord struct {
+	Header *tar.Header
+	CopyTo func(context.Context, io.Writer) (int64, error)
+}
+
+func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord {
+	path := path.Join("blobs", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     path,
+			Mode:     0444,
+			Size:     desc.Size,
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			r, err := cs.ReaderAt(ctx, desc)
+			if err != nil {
+				return 0, errors.Wrap(err, "failed to get reader")
+			}
+			defer r.Close()
+
+			// Verify digest
+			dgstr := desc.Digest.Algorithm().Digester()
+
+			n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
+			if err != nil {
+				return 0, errors.Wrap(err, "failed to copy to tar")
+			}
+			if dgstr.Digest() != desc.Digest {
+				return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())
+			}
+			return n, nil
+		},
+	}
+}
+
+func directoryRecord(name string, mode int64) tarRecord {
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     name,
+			Mode:     mode,
+			Typeflag: tar.TypeDir,
+		},
+	}
+}
+
+func ociLayoutFile(version string) tarRecord {
+	if version == "" {
+		version = ocispec.ImageLayoutVersion
+	}
+	layout := ocispec.ImageLayout{
+		Version: version,
+	}
+
+	b, err := json.Marshal(layout)
+	if err != nil {
+		panic(err)
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     ocispec.ImageLayoutFile,
+			Mode:     0444,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}
+
+}
+
+func ociIndexRecord(manifests []ocispec.Descriptor) tarRecord {
+	index := ocispec.Index{
+		Versioned: ocispecs.Versioned{
+			SchemaVersion: 2,
+		},
+		Manifests: manifests,
+	}
+
+	b, err := json.Marshal(index)
+	if err != nil {
+		panic(err)
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     "index.json",
+			Mode:     0644,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}
+}
+
+func manifestsRecord(ctx context.Context, store content.Provider, manifests map[string]ocispec.Descriptor) (tarRecord, error) {
+	type mfst struct {
+		Config   string
+		RepoTags []string
+		Layers   []string
+	}
+
+	images := map[digest.Digest]mfst{}
+	for name, m := range manifests {
+		p, err := content.ReadBlob(ctx, store, m)
+		if err != nil {
+			return tarRecord{}, err
+		}
+
+		var manifest ocispec.Manifest
+		if err := json.Unmarshal(p, &manifest); err != nil {
+			return tarRecord{}, err
+		}
+		if err := manifest.Config.Digest.Validate(); err != nil {
+			return tarRecord{}, errors.Wrapf(err, "invalid manifest %q", m.Digest)
+		}
+
+		nname, err := familiarizeReference(name)
+		if err != nil {
+			return tarRecord{}, err
+		}
+
+		dgst := manifest.Config.Digest
+		mf, ok := images[dgst]
+		if !ok {
+			mf.Config = path.Join("blobs", dgst.Algorithm().String(), dgst.Encoded())
+			for _, l := range manifest.Layers {
+				path := path.Join("blobs", l.Digest.Algorithm().String(), l.Digest.Encoded())
+				mf.Layers = append(mf.Layers, path)
+			}
+		}
+
+		mf.RepoTags = append(mf.RepoTags, nname)
+
+		images[dgst] = mf
+	}
+
+	var mfsts []mfst
+	for _, mf := range images {
+		mfsts = append(mfsts, mf)
+	}
+
+	b, err := json.Marshal(mfsts)
+	if err != nil {
+		return tarRecord{}, err
+	}
+
+	return tarRecord{
+		Header: &tar.Header{
+			Name:     "manifest.json",
+			Mode:     0644,
+			Size:     int64(len(b)),
+			Typeflag: tar.TypeReg,
+		},
+		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
+			n, err := w.Write(b)
+			return int64(n), err
+		},
+	}, nil
+}
+
+func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
+	sort.Slice(records, func(i, j int) bool {
+		return records[i].Header.Name < records[j].Header.Name
+	})
+
+	var last string
+	for _, record := range records {
+		if record.Header.Name == last {
+			continue
+		}
+		last = record.Header.Name
+		if err := tw.WriteHeader(record.Header); err != nil {
+			return err
+		}
+		if record.CopyTo != nil {
+			n, err := record.CopyTo(ctx, tw)
+			if err != nil {
+				return err
+			}
+			if n != record.Header.Size {
+				return errors.Errorf("unexpected copy size for %s", record.Header.Name)
+			}
+		} else if record.Header.Size > 0 {
+			return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
+		}
+	}
+	return nil
+}

images/archive/reference.go

@@ -77,6 +77,16 @@ func normalizeReference(ref string) (string, error) {
 	return normalized.String(), nil
 }
 
+func familiarizeReference(ref string) (string, error) {
+	named, err := reference.ParseNormalizedNamed(ref)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse %q", ref)
+	}
+	named = reference.TagNameOnly(named)
+
+	return reference.FamiliarString(named), nil
+}
+
 // DigestTranslator creates a digest reference by adding the
 // digest to an image name
 func DigestTranslator(prefix string) func(digest.Digest) string {

images/oci/exporter.go

@@ -1,241 +0,0 @@
-/*
-   Copyright The containerd Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-*/
-
-package oci
-
-import (
-	"archive/tar"
-	"context"
-	"encoding/json"
-	"io"
-	"sort"
-
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/platforms"
-	ocispecs "github.com/opencontainers/image-spec/specs-go"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-)
-
-// V1Exporter implements OCI Image Spec v1.
-// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
-//
-// TODO(AkihiroSuda): add V1Exporter{TranslateMediaTypes: true} that transforms media types,
-//                    e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
-//                         -> application/vnd.oci.image.layer.v1.tar+gzip
-type V1Exporter struct {
-	AllPlatforms bool
-}
-
-// V1ExporterOpt allows the caller to set additional options to a new V1Exporter
-type V1ExporterOpt func(c *V1Exporter) error
-
-// DefaultV1Exporter return a default V1Exporter pointer
-func DefaultV1Exporter() *V1Exporter {
-	return &V1Exporter{
-		AllPlatforms: false,
-	}
-}
-
-// ResolveV1ExportOpt return a new V1Exporter with V1ExporterOpt
-func ResolveV1ExportOpt(opts ...V1ExporterOpt) (*V1Exporter, error) {
-	exporter := DefaultV1Exporter()
-	for _, o := range opts {
-		if err := o(exporter); err != nil {
-			return exporter, err
-		}
-	}
-	return exporter, nil
-}
-
-// WithAllPlatforms set V1Exporter`s AllPlatforms option
-func WithAllPlatforms(allPlatforms bool) V1ExporterOpt {
-	return func(c *V1Exporter) error {
-		c.AllPlatforms = allPlatforms
-		return nil
-	}
-}
-
-// Export implements Exporter.
-func (oe *V1Exporter) Export(ctx context.Context, store content.Provider, desc ocispec.Descriptor, writer io.Writer) error {
-	tw := tar.NewWriter(writer)
-	defer tw.Close()
-
-	records := []tarRecord{
-		ociLayoutFile(""),
-		ociIndexRecord(desc),
-	}
-
-	algorithms := map[string]struct{}{}
-	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-		records = append(records, blobRecord(store, desc))
-		algorithms[desc.Digest.Algorithm().String()] = struct{}{}
-		return nil, nil
-	}
-
-	childrenHandler := images.ChildrenHandler(store)
-
-	if !oe.AllPlatforms {
-		// get local default platform to fetch image manifest
-		childrenHandler = images.FilterPlatforms(childrenHandler, platforms.Any(platforms.DefaultSpec()))
-	}
-
-	handlers := images.Handlers(
-		childrenHandler,
-		images.HandlerFunc(exportHandler),
-	)
-
-	// Walk sequentially since the number of fetchs is likely one and doing in
-	// parallel requires locking the export handler
-	if err := images.Walk(ctx, handlers, desc); err != nil {
-		return err
-	}
-
-	if len(algorithms) > 0 {
-		records = append(records, directoryRecord("blobs/", 0755))
-		for alg := range algorithms {
-			records = append(records, directoryRecord("blobs/"+alg+"/", 0755))
-		}
-	}
-
-	return writeTar(ctx, tw, records)
-}
-
-type tarRecord struct {
-	Header *tar.Header
-	CopyTo func(context.Context, io.Writer) (int64, error)
-}
-
-func blobRecord(cs content.Provider, desc ocispec.Descriptor) tarRecord {
-	path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex()
-	return tarRecord{
-		Header: &tar.Header{
-			Name:     path,
-			Mode:     0444,
-			Size:     desc.Size,
-			Typeflag: tar.TypeReg,
-		},
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
-			r, err := cs.ReaderAt(ctx, desc)
-			if err != nil {
-				return 0, errors.Wrap(err, "failed to get reader")
-			}
-			defer r.Close()
-
-			// Verify digest
-			dgstr := desc.Digest.Algorithm().Digester()
-
-			n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r))
-			if err != nil {
-				return 0, errors.Wrap(err, "failed to copy to tar")
-			}
-			if dgstr.Digest() != desc.Digest {
-				return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest())
-			}
-			return n, nil
-		},
-	}
-}
-
-func directoryRecord(name string, mode int64) tarRecord {
-	return tarRecord{
-		Header: &tar.Header{
-			Name:     name,
-			Mode:     mode,
-			Typeflag: tar.TypeDir,
-		},
-	}
-}
-
-func ociLayoutFile(version string) tarRecord {
-	if version == "" {
-		version = ocispec.ImageLayoutVersion
-	}
-	layout := ocispec.ImageLayout{
-		Version: version,
-	}
-
-	b, err := json.Marshal(layout)
-	if err != nil {
-		panic(err)
-	}
-
-	return tarRecord{
-		Header: &tar.Header{
-			Name:     ocispec.ImageLayoutFile,
-			Mode:     0444,
-			Size:     int64(len(b)),
-			Typeflag: tar.TypeReg,
-		},
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
-			n, err := w.Write(b)
-			return int64(n), err
-		},
-	}
-
-}
-
-func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord {
-	index := ocispec.Index{
-		Versioned: ocispecs.Versioned{
-			SchemaVersion: 2,
-		},
-		Manifests: manifests,
-	}
-
-	b, err := json.Marshal(index)
-	if err != nil {
-		panic(err)
-	}
-
-	return tarRecord{
-		Header: &tar.Header{
-			Name:     "index.json",
-			Mode:     0644,
-			Size:     int64(len(b)),
-			Typeflag: tar.TypeReg,
-		},
-		CopyTo: func(ctx context.Context, w io.Writer) (int64, error) {
-			n, err := w.Write(b)
-			return int64(n), err
-		},
-	}
-}
-
-func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
-	sort.Slice(records, func(i, j int) bool {
-		return records[i].Header.Name < records[j].Header.Name
-	})
-
-	for _, record := range records {
-		if err := tw.WriteHeader(record.Header); err != nil {
-			return err
-		}
-		if record.CopyTo != nil {
-			n, err := record.CopyTo(ctx, tw)
-			if err != nil {
-				return err
-			}
-			if n != record.Header.Size {
-				return errors.Errorf("unexpected copy size for %s", record.Header.Name)
-			}
-		} else if record.Header.Size > 0 {
-			return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name)
-		}
-	}
-	return nil
-}

import.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"strings"
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
@@ -130,16 +131,12 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 		}
 
 		for _, m := range idx.Manifests {
-			if ref := m.Annotations[ocispec.AnnotationRefName]; ref != "" {
+			names := imageNames(m.Annotations, iopts.imageRefT)
-				if iopts.imageRefT != nil {
+			for _, name := range names {
-					ref = iopts.imageRefT(ref)
+				imgs = append(imgs, images.Image{
-				}
+					Name:   name,
-				if ref != "" {
+					Target: m,
-					imgs = append(imgs, images.Image{
+				})
-						Name:   ref,
-						Target: m,
-					})
-				}
 			}
 			if iopts.dgstRefT != nil {
 				ref := iopts.dgstRefT(m.Digest)
@@ -178,3 +175,35 @@ func (c *Client) Import(ctx context.Context, reader io.Reader, opts ...ImportOpt
 
 	return imgs, nil
 }
+
+func imageNames(annotations map[string]string, ociCleanup func(string) string) []string {
+	var names []string
+	for k, v := range annotations {
+		if k == ocispec.AnnotationRefName {
+			if ociCleanup != nil {
+				v = ociCleanup(v)
+			}
+			if v != "" {
+				names = appendSorted(names, v)
+			}
+		} else if k == images.AnnotationImageName || strings.HasPrefix(k, images.AnnotationImageNamePrefix) {
+			names = appendSorted(names, v)
+
+		}
+	}
+	return names
+}
+
+func appendSorted(arr []string, s string) []string {
+	for i, c := range arr {
+		if s < c {
+			arr = append(arr, "")
+			copy(arr[i+1:], arr[i:])
+			arr[i] = s
+			return arr
+		} else if s == c {
+			return arr
+		}
+	}
+	return append(arr, s)
+}

import_test.go

@@ -17,6 +17,7 @@
 package containerd
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"io"
@@ -35,9 +36,9 @@ import (
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// TestOCIExportAndImport exports testImage as a tar stream,
+// TestExportAndImport exports testImage as a tar stream,
 // and import the tar stream as a new image.
-func TestOCIExportAndImport(t *testing.T) {
+func TestExportAndImport(t *testing.T) {
 	// TODO: support windows
 	if testing.Short() || runtime.GOOS == "windows" {
 		t.Skip()
@@ -51,12 +52,13 @@ func TestOCIExportAndImport(t *testing.T) {
 	}
 	defer client.Close()
 
-	pulled, err := client.Fetch(ctx, testImage)
+	_, err = client.Fetch(ctx, testImage)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	exported, err := client.Export(ctx, pulled.Target)
+	wb := bytes.NewBuffer(nil)
+	err = client.Export(ctx, wb, archive.WithAllPlatforms(), archive.WithImage(client.ImageService(), testImage))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -64,12 +66,15 @@ func TestOCIExportAndImport(t *testing.T) {
 	opts := []ImportOpt{
 		WithImageRefTranslator(archive.AddRefPrefix("foo/bar")),
 	}
-	imgrecs, err := client.Import(ctx, exported, opts...)
+	imgrecs, err := client.Import(ctx, bytes.NewReader(wb.Bytes()), opts...)
 	if err != nil {
 		t.Fatalf("Import failed: %+v", err)
 	}
 
 	for _, imgrec := range imgrecs {
+		if imgrec.Name == testImage {
+			continue
+		}
 		err = client.ImageService().Delete(ctx, imgrec.Name)
 		if err != nil {
 			t.Fatal(err)