Merge pull request #1880 from AkihiroSuda/refactor-importer

importer: refactor and fix GC

client.go

@@ -30,7 +30,6 @@ import (
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/plugin"
-	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/remotes/docker/schema1"
@@ -503,95 +502,27 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
 	}, nil
 }
 
-type imageFormat string
-
-const (
-	ociImageFormat imageFormat = "oci"
-)
-
 type importOpts struct {
-	format    imageFormat
-	refObject string
-	labels    map[string]string
 }
 
 // ImportOpt allows the caller to specify import specific options
 type ImportOpt func(c *importOpts) error
 
-// WithImportLabel sets a label to be associated with an imported image
+func resolveImportOpt(opts ...ImportOpt) (importOpts, error) {
-func WithImportLabel(key, value string) ImportOpt {
-	return func(opts *importOpts) error {
-		if opts.labels == nil {
-			opts.labels = make(map[string]string)
-		}
-
-		opts.labels[key] = value
-		return nil
-	}
-}
-
-// WithImportLabels associates a set of labels to an imported image
-func WithImportLabels(labels map[string]string) ImportOpt {
-	return func(opts *importOpts) error {
-		if opts.labels == nil {
-			opts.labels = make(map[string]string)
-		}
-
-		for k, v := range labels {
-			opts.labels[k] = v
-		}
-		return nil
-	}
-}
-
-// WithOCIImportFormat sets the import format for an OCI image format
-func WithOCIImportFormat() ImportOpt {
-	return func(c *importOpts) error {
-		if c.format != "" {
-			return errors.New("format already set")
-		}
-		c.format = ociImageFormat
-		return nil
-	}
-}
-
-// WithRefObject specifies the ref object to import.
-// If refObject is empty, it is copied from the ref argument of Import().
-func WithRefObject(refObject string) ImportOpt {
-	return func(c *importOpts) error {
-		c.refObject = refObject
-		return nil
-	}
-}
-
-func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
 	var iopts importOpts
 	for _, o := range opts {
 		if err := o(&iopts); err != nil {
 			return iopts, err
 		}
 	}
-	// use OCI as the default format
-	if iopts.format == "" {
-		iopts.format = ociImageFormat
-	}
-	// if refObject is not explicitly specified, use the one specified in ref
-	if iopts.refObject == "" {
-		refSpec, err := reference.Parse(ref)
-		if err != nil {
-			return iopts, err
-		}
-		iopts.refObject = refSpec.Object
-	}
 	return iopts, nil
 }
 
 // Import imports an image from a Tar stream using reader.
-// OCI format is assumed by default.
+// Caller needs to specify importer. Future version may use oci.v1 as the default.
-//
+// Note that unreferrenced blobs may be imported to the content store as well.
-// Note that unreferenced blobs are imported to the content store as well.
+func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) {
-func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
+	_, err := resolveImportOpt(opts...) // unused now
-	iopts, err := resolveImportOpt(ref, opts...)
 	if err != nil {
 		return nil, err
 	}
@@ -602,58 +533,66 @@ func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts
 	}
 	defer done()
 
-	switch iopts.format {
+	imgrecs, err := importer.Import(ctx, c.ContentStore(), reader)
-	case ociImageFormat:
+	if err != nil {
-		return c.importFromOCITar(ctx, ref, reader, iopts)
+		// is.Update() is not called on error
-	default:
+		return nil, err
-		return nil, errors.Errorf("unsupported format: %s", iopts.format)
 	}
+
+	is := c.ImageService()
+	var images []Image
+	for _, imgrec := range imgrecs {
+		if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
+			if !errdefs.IsNotFound(err) {
+				return nil, err
+			}
+
+			created, err := is.Create(ctx, imgrec)
+			if err != nil {
+				return nil, err
+			}
+
+			imgrec = created
+		} else {
+			imgrec = updated
+		}
+
+		images = append(images, &image{
+			client: c,
+			i:      imgrec,
+		})
+	}
+	return images, nil
 }
 
 type exportOpts struct {
-	format imageFormat
 }
 
-// ExportOpt allows callers to set export options
+// ExportOpt allows the caller to specify export-specific options
 type ExportOpt func(c *exportOpts) error
 
-// WithOCIExportFormat sets the OCI image format as the export target
+func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) {
-func WithOCIExportFormat() ExportOpt {
+	var eopts exportOpts
-	return func(c *exportOpts) error {
+	for _, o := range opts {
-		if c.format != "" {
+		if err := o(&eopts); err != nil {
-			return errors.New("format already set")
+			return eopts, err
 		}
-		c.format = ociImageFormat
-		return nil
 	}
+	return eopts, nil
 }
 
-// TODO: add WithMediaTypeTranslation that transforms media types according to the format.
-// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
-//      -> application/vnd.oci.image.layer.v1.tar+gzip
-
 // Export exports an image to a Tar stream.
 // OCI format is used by default.
 // It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
-func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
+// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream.
-	var eopts exportOpts
+func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
-	for _, o := range opts {
+	_, err := resolveExportOpt(opts...) // unused now
-		if err := o(&eopts); err != nil {
+	if err != nil {
-			return nil, err
+		return nil, err
-		}
-	}
-	// use OCI as the default format
-	if eopts.format == "" {
-		eopts.format = ociImageFormat
 	}
 	pr, pw := io.Pipe()
-	switch eopts.format {
+	go func() {
-	case ociImageFormat:
+		pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw))
-		go func() {
+	}()
-			pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts))
-		}()
-	default:
-		return nil, errors.Errorf("unsupported format: %s", eopts.format)
-	}
 	return pr, nil
 }

cmd/ctr/commands/images/export.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/containerd/containerd/cmd/ctr/commands"
+	oci "github.com/containerd/containerd/images/oci"
 	"github.com/containerd/containerd/reference"
 	digest "github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -13,11 +14,14 @@ import (
 )
 
 var exportCommand = cli.Command{
-	Name:        "export",
+	Name:      "export",
-	Usage:       "export an image",
+	Usage:     "export an image",
-	ArgsUsage:   "[flags] <out> <image>",
+	ArgsUsage: "[flags] <out> <image>",
-	Description: "export an image to a tar stream",
+	Description: `Export an image to a tar stream.
+Currently, only OCI format is supported.
+`,
 	Flags: []cli.Flag{
+		// TODO(AkihiroSuda): make this map[string]string as in moby/moby#33355?
 		cli.StringFlag{
 			Name:  "oci-ref-name",
 			Value: "",
@@ -78,7 +82,7 @@ var exportCommand = cli.Command{
 				return nil
 			}
 		}
-		r, err := client.Export(ctx, desc)
+		r, err := client.Export(ctx, &oci.V1Exporter{}, desc)
 		if err != nil {
 			return err
 		}

cmd/ctr/commands/images/import.go

@@ -5,37 +5,66 @@ import (
 	"io"
 	"os"
 
-	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/containerd/images"
+	oci "github.com/containerd/containerd/images/oci"
 	"github.com/containerd/containerd/log"
 	"github.com/urfave/cli"
 )
 
 var importCommand = cli.Command{
-	Name:        "import",
+	Name:      "import",
-	Usage:       "import an image",
+	Usage:     "import images",
-	ArgsUsage:   "[flags] <ref> <in>",
+	ArgsUsage: "[flags] <in>",
-	Description: "import an image from a tar stream",
+	Description: `Import images from a tar stream.
-	Flags: []cli.Flag{
+Implemented formats:
+- oci.v1     (default)
+
+
+For oci.v1 format, you need to specify --oci-name because an OCI archive contains image refs (tags)
+but does not contain the base image name.
+
+e.g.
+  $ ctr images import --format oci.v1 --oci-name foo/bar foobar.tar
+
+If foobar.tar contains an OCI ref named "latest" and anonymous ref "sha256:deadbeef", the command will create
+"foo/bar:latest" and "foo/bar@sha256:deadbeef" images in the containerd store.
+`,
+	Flags: append([]cli.Flag{
 		cli.StringFlag{
-			Name:  "ref-object",
+			Name:  "format",
-			Value: "",
+			Value: "oci.v1",
-			Usage: "reference object e.g. tag@digest (default: use the object specified in ref)",
+			Usage: "image format. See DESCRIPTION.",
 		},
-		commands.LabelFlag,
+		cli.StringFlag{
-	},
+			Name:  "oci-name",
+			Value: "unknown/unknown",
+			Usage: "prefix added to either oci.v1 ref annotation or digest",
+		},
+		// TODO(AkihiroSuda): support commands.LabelFlag (for all children objects)
+	}, commands.SnapshotterFlags...),
+
 	Action: func(context *cli.Context) error {
 		var (
-			ref       = context.Args().First()
+			in            = context.Args().First()
-			in        = context.Args().Get(1)
+			imageImporter images.Importer
-			refObject = context.String("ref-object")
-			labels    = commands.LabelArgs(context.StringSlice("label"))
 		)
+
+		switch format := context.String("format"); format {
+		case "oci.v1":
+			imageImporter = &oci.V1Importer{
+				ImageName: context.String("oci-name"),
+			}
+		default:
+			return fmt.Errorf("unknown format %s", format)
+		}
+
 		client, ctx, cancel, err := commands.NewClient(context)
 		if err != nil {
 			return err
 		}
 		defer cancel()
+
 		var r io.ReadCloser
 		if in == "-" {
 			r = os.Stdin
@@ -45,12 +74,7 @@ var importCommand = cli.Command{
 				return err
 			}
 		}
-		img, err := client.Import(ctx,
+		imgs, err := client.Import(ctx, imageImporter, r)
-			ref,
-			r,
-			containerd.WithRefObject(refObject),
-			containerd.WithImportLabels(labels),
-		)
 		if err != nil {
 			return err
 		}
@@ -58,12 +82,17 @@ var importCommand = cli.Command{
 			return err
 		}
 
-		log.G(ctx).WithField("image", ref).Debug("unpacking")
+		log.G(ctx).Debugf("unpacking %d images", len(imgs))
 
-		// TODO: Show unpack status
+		for _, img := range imgs {
-		fmt.Printf("unpacking %s...", img.Target().Digest)
+			// TODO: Show unpack status
-		err = img.Unpack(ctx, context.String("snapshotter"))
+			fmt.Printf("unpacking %s (%s)...", img.Name(), img.Target().Digest)
-		fmt.Println("done")
+			err = img.Unpack(ctx, context.String("snapshotter"))
-		return err
+			if err != nil {
+				return err
+			}
+			fmt.Println("done")
+		}
+		return nil
 	},
 }

export_test.go

@@ -5,10 +5,12 @@ import (
 	"io"
 	"runtime"
 	"testing"
+
+	"github.com/containerd/containerd/images/oci"
 )
 
-// TestExport exports testImage as a tar stream
+// TestOCIExport exports testImage as a tar stream
-func TestExport(t *testing.T) {
+func TestOCIExport(t *testing.T) {
 	// TODO: support windows
 	if testing.Short() || runtime.GOOS == "windows" {
 		t.Skip()
@@ -26,8 +28,7 @@ func TestExport(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
+	exportedStream, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target())
-	exportedStream, err := client.Export(ctx, pulled.Target())
 	if err != nil {
 		t.Fatal(err)
 	}

images/importexport.go

@@ -0,0 +1,21 @@
+package images
+
+import (
+	"context"
+	"io"
+
+	"github.com/containerd/containerd/content"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// Importer is the interface for image importer.
+type Importer interface {
+	// Import imports an image from a tar stream.
+	Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error)
+}
+
+// Exporter is the interface for image exporter.
+type Exporter interface {
+	// Export exports an image to a tar stream.
+	Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error
+}

images/oci/exporter.go

@@ -1,4 +1,4 @@
-package containerd
+package oci
 
 import (
 	"archive/tar"
@@ -15,7 +15,17 @@ import (
 	"github.com/pkg/errors"
 )
 
-func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error {
+// V1Exporter implements OCI Image Spec v1.
+// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
+//
+// TODO(AkihiroSuda): add V1Exporter{TranslateMediaTypes: true} that transforms media types,
+//                    e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
+//                         -> application/vnd.oci.image.layer.v1.tar+gzip
+type V1Exporter struct {
+}
+
+// Export implements Exporter.
+func (oe *V1Exporter) Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error {
 	tw := tar.NewWriter(writer)
 	defer tw.Close()
 
@@ -24,16 +34,15 @@ func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, wr
 		ociIndexRecord(desc),
 	}
 
-	cs := c.ContentStore()
 	algorithms := map[string]struct{}{}
 	exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
-		records = append(records, blobRecord(cs, desc))
+		records = append(records, blobRecord(store, desc))
 		algorithms[desc.Digest.Algorithm().String()] = struct{}{}
 		return nil, nil
 	}
 
 	handlers := images.Handlers(
-		images.ChildrenHandler(cs, platforms.Default()),
+		images.ChildrenHandler(store, platforms.Default()),
 		images.HandlerFunc(exportHandler),
 	)
 
@@ -155,7 +164,9 @@ func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord {
 }
 
 func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
-	sort.Sort(tarRecordsByName(records))
+	sort.Slice(records, func(i, j int) bool {
+		return records[i].Header.Name < records[j].Header.Name
+	})
 
 	for _, record := range records {
 		if err := tw.WriteHeader(record.Header); err != nil {
@@ -175,15 +186,3 @@ func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error {
 	}
 	return nil
 }
-
-type tarRecordsByName []tarRecord
-
-func (t tarRecordsByName) Len() int {
-	return len(t)
-}
-func (t tarRecordsByName) Swap(i, j int) {
-	t[i], t[j] = t[j], t[i]
-}
-func (t tarRecordsByName) Less(i, j int) bool {
-	return t[i].Header.Name < t[j].Header.Name
-}

images/oci/importer.go

@@ -0,0 +1,188 @@
+// Package oci provides the importer and the exporter for OCI Image Spec.
+package oci
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// V1Importer implements OCI Image Spec v1.
+type V1Importer struct {
+	// ImageName is preprended to either `:` + OCI ref name or `@` + digest (for anonymous refs).
+	// This field is mandatory atm, but may change in the future. maybe ref map[string]string as in moby/moby#33355
+	ImageName string
+}
+
+var _ images.Importer = &V1Importer{}
+
+// Import implements Importer.
+func (oi *V1Importer) Import(ctx context.Context, store content.Store, reader io.Reader) ([]images.Image, error) {
+	if oi.ImageName == "" {
+		return nil, errors.New("ImageName not set")
+	}
+	tr := tar.NewReader(reader)
+	var imgrecs []images.Image
+	foundIndexJSON := false
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
+			continue
+		}
+		hdrName := path.Clean(hdr.Name)
+		if hdrName == "index.json" {
+			if foundIndexJSON {
+				return nil, errors.New("duplicated index.json")
+			}
+			foundIndexJSON = true
+			imgrecs, err = onUntarIndexJSON(tr, oi.ImageName)
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+		if strings.HasPrefix(hdrName, "blobs/") {
+			if err := onUntarBlob(ctx, tr, store, hdrName, hdr.Size); err != nil {
+				return nil, err
+			}
+		}
+	}
+	if !foundIndexJSON {
+		return nil, errors.New("no index.json found")
+	}
+	for _, img := range imgrecs {
+		err := setGCRefContentLabels(ctx, store, img.Target)
+		if err != nil {
+			return imgrecs, err
+		}
+	}
+	// FIXME(AkihiroSuda): set GC labels for unreferrenced blobs (i.e. with unknown media types)?
+	return imgrecs, nil
+}
+
+func onUntarIndexJSON(r io.Reader, imageName string) ([]images.Image, error) {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	var idx ocispec.Index
+	if err := json.Unmarshal(b, &idx); err != nil {
+		return nil, err
+	}
+	var imgrecs []images.Image
+	for _, m := range idx.Manifests {
+		ref, err := normalizeImageRef(imageName, m)
+		if err != nil {
+			return nil, err
+		}
+		imgrecs = append(imgrecs, images.Image{
+			Name:   ref,
+			Target: m,
+		})
+	}
+	return imgrecs, nil
+}
+
+func normalizeImageRef(imageName string, manifest ocispec.Descriptor) (string, error) {
+	digest := manifest.Digest
+	if digest == "" {
+		return "", errors.Errorf("manifest with empty digest: %v", manifest)
+	}
+	ociRef := manifest.Annotations[ocispec.AnnotationRefName]
+	if ociRef == "" {
+		return imageName + "@" + digest.String(), nil
+	}
+	return imageName + ":" + ociRef, nil
+}
+
+func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
+	// name is like "blobs/sha256/deadbeef"
+	split := strings.Split(name, "/")
+	if len(split) != 3 {
+		return errors.Errorf("unexpected name: %q", name)
+	}
+	algo := digest.Algorithm(split[1])
+	if !algo.Available() {
+		return errors.Errorf("unsupported algorithm: %s", algo)
+	}
+	dgst := digest.NewDigestFromHex(algo.String(), split[2])
+	return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
+}
+
+// GetChildrenDescriptors returns children blob descriptors for the following supported types:
+// - images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest
+// - images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex
+func GetChildrenDescriptors(r io.Reader, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+	switch desc.MediaType {
+	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
+		var manifest ocispec.Manifest
+		if err := json.NewDecoder(r).Decode(&manifest); err != nil {
+			return nil, err
+		}
+		return append([]ocispec.Descriptor{manifest.Config}, manifest.Layers...), nil
+	case images.MediaTypeDockerSchema2ManifestList, ocispec.MediaTypeImageIndex:
+		var index ocispec.Index
+		if err := json.NewDecoder(r).Decode(&index); err != nil {
+			return nil, err
+		}
+		return index.Manifests, nil
+	}
+	return nil, nil
+}
+
+func setGCRefContentLabels(ctx context.Context, store content.Store, desc ocispec.Descriptor) error {
+	info, err := store.Info(ctx, desc.Digest)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			// when the archive is created from multi-arch image,
+			// it may contain only blobs for a certain platform.
+			// So ErrNotFound (on manifest list) is expected here.
+			return nil
+		}
+		return err
+	}
+	ra, err := store.ReaderAt(ctx, desc.Digest)
+	if err != nil {
+		return err
+	}
+	defer ra.Close()
+	r := content.NewReader(ra)
+	children, err := GetChildrenDescriptors(r, desc)
+	if err != nil {
+		return err
+	}
+	if info.Labels == nil {
+		info.Labels = map[string]string{}
+	}
+	for i, child := range children {
+		// Note: child blob is not guaranteed to be written to the content store. (multi-arch)
+		info.Labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = child.Digest.String()
+	}
+	if _, err := store.Update(ctx, info, "labels"); err != nil {
+		return err
+	}
+	for _, child := range children {
+		if err := setGCRefContentLabels(ctx, store, child); err != nil {
+			return err
+		}
+	}
+	return nil
+}

images/oci/importer_test.go

@@ -0,0 +1,37 @@
+package oci
+
+import (
+	"testing"
+
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNormalizeImageRef(t *testing.T) {
+	imageBaseName := "foo/bar"
+	for _, test := range []struct {
+		input  ocispec.Descriptor
+		expect string
+	}{
+		{
+			input: ocispec.Descriptor{
+				Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
+			},
+			expect: "foo/bar@sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77",
+		},
+		{
+			input: ocispec.Descriptor{
+				Digest: digest.Digest("sha256:e22e93af8657d43d7f204b93d69604aeacf273f71d2586288cde312808c0ec77"),
+				Annotations: map[string]string{
+					ocispec.AnnotationRefName: "latest",
+				},
+			},
+			expect: "foo/bar:latest", // no @digest for simplicity
+		},
+	} {
+		normalized, err := normalizeImageRef(imageBaseName, test.input)
+		assert.NoError(t, err)
+		assert.Equal(t, test.expect, normalized)
+	}
+}

import.go

@@ -1,120 +0,0 @@
-package containerd
-
-import (
-	"archive/tar"
-	"context"
-	"encoding/json"
-	"io"
-	"io/ioutil"
-	"strings"
-
-	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/reference"
-	digest "github.com/opencontainers/go-digest"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
-)
-
-func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) {
-	tag, dgst := reference.SplitObject(refObject)
-	if tag == "" && dgst == "" {
-		return nil, errors.Errorf("unexpected object: %q", refObject)
-	}
-	for _, m := range idx.Manifests {
-		if m.Digest == dgst {
-			return &m, nil
-		}
-		annot, ok := m.Annotations[ocispec.AnnotationRefName]
-		if ok && annot == tag && tag != "" {
-			return &m, nil
-		}
-	}
-	return nil, errors.Errorf("not found: %q", refObject)
-}
-
-func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) {
-	tr := tar.NewReader(reader)
-	store := c.ContentStore()
-	var desc *ocispec.Descriptor
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return nil, err
-		}
-		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
-			continue
-		}
-		if hdr.Name == "index.json" {
-			desc, err = onUntarIndexJSON(tr, iopts.refObject)
-			if err != nil {
-				return nil, err
-			}
-			continue
-		}
-		if strings.HasPrefix(hdr.Name, "blobs/") {
-			if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil {
-				return nil, err
-			}
-		}
-	}
-	if desc == nil {
-		return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject)
-	}
-	imgrec := images.Image{
-		Name:   ref,
-		Target: *desc,
-		Labels: iopts.labels,
-	}
-	is := c.ImageService()
-	if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
-		if !errdefs.IsNotFound(err) {
-			return nil, err
-		}
-
-		created, err := is.Create(ctx, imgrec)
-		if err != nil {
-			return nil, err
-		}
-
-		imgrec = created
-	} else {
-		imgrec = updated
-	}
-
-	img := &image{
-		client: c,
-		i:      imgrec,
-	}
-	return img, nil
-}
-
-func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) {
-	b, err := ioutil.ReadAll(r)
-	if err != nil {
-		return nil, err
-	}
-	var idx ocispec.Index
-	if err := json.Unmarshal(b, &idx); err != nil {
-		return nil, err
-	}
-	return resolveOCIIndex(idx, refObject)
-}
-
-func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
-	// name is like "blobs/sha256/deadbeef"
-	split := strings.Split(name, "/")
-	if len(split) != 3 {
-		return errors.Errorf("unexpected name: %q", name)
-	}
-	algo := digest.Algorithm(split[1])
-	if !algo.Available() {
-		return errors.Errorf("unsupported algorithm: %s", algo)
-	}
-	dgst := digest.NewDigestFromHex(algo.String(), split[2])
-	return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
-}

import_test.go

@@ -3,11 +3,13 @@ package containerd
 import (
 	"runtime"
 	"testing"
+
+	"github.com/containerd/containerd/images/oci"
 )
 
-// TestExportAndImport exports testImage as a tar stream,
+// TestOCIExportAndImport exports testImage as a tar stream,
 // and import the tar stream as a new image.
-func TestExportAndImport(t *testing.T) {
+func TestOCIExportAndImport(t *testing.T) {
 	// TODO: support windows
 	if testing.Short() || runtime.GOOS == "windows" {
 		t.Skip()
@@ -26,19 +28,20 @@ func TestExportAndImport(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	exported, err := client.Export(ctx, pulled.Target())
+	exported, err := client.Export(ctx, &oci.V1Exporter{}, pulled.Target())
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	importRef := "test/export-and-import:tmp"
+	imgrecs, err := client.Import(ctx, &oci.V1Importer{ImageName: "foo/bar:"}, exported)
-	_, err = client.Import(ctx, importRef, exported, WithRefObject("@"+pulled.Target().Digest.String()))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	err = client.ImageService().Delete(ctx, importRef)
+	for _, imgrec := range imgrecs {
-	if err != nil {
+		err = client.ImageService().Delete(ctx, imgrec.Name())
-		t.Fatal(err)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
 }

remotes/handlers.go

@@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc
 func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) {
 	var childrenF func(r io.Reader) ([]ocispec.Descriptor, error)
 
+	// TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors?
 	switch desc.MediaType {
 	case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
 		childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) {