client: add Import() and Export() for importing/exporting image in OCI format

Export as a tar (Note: "-" can be used for stdout):

    $ ctr images export /tmp/oci-busybox.tar docker.io/library/busybox:latest

Import a tar (Note: "-" can be used for stdin):

    $ ctr images import foo/new:latest /tmp/oci-busybox.tar

Note: media types are not converted at the moment: e.g.
  application/vnd.docker.image.rootfs.diff.tar.gzip
  -> application/vnd.oci.image.layer.v1.tar+gzip

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>

client.go

@@ -3,6 +3,7 @@ package containerd
 import (
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -25,6 +26,7 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/reference"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/remotes/docker/schema1"
@@ -552,3 +554,120 @@ func (c *Client) Version(ctx context.Context) (Version, error) {
 		Revision: response.Revision,
 	}, nil
 }
+
+type imageFormat string
+
+const (
+	ociImageFormat imageFormat = "oci"
+)
+
+type importOpts struct {
+	format    imageFormat
+	refObject string
+}
+
+type ImportOpt func(c *importOpts) error
+
+func WithOCIImportFormat() ImportOpt {
+	return func(c *importOpts) error {
+		if c.format != "" {
+			return errors.New("format already set")
+		}
+		c.format = ociImageFormat
+		return nil
+	}
+}
+
+// WithRefObject specifies the ref object to import.
+// If refObject is empty, it is copied from the ref argument of Import().
+func WithRefObject(refObject string) ImportOpt {
+	return func(c *importOpts) error {
+		c.refObject = refObject
+		return nil
+	}
+}
+
+func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) {
+	var iopts importOpts
+	for _, o := range opts {
+		if err := o(&iopts); err != nil {
+			return iopts, err
+		}
+	}
+	// use OCI as the default format
+	if iopts.format == "" {
+		iopts.format = ociImageFormat
+	}
+	// if refObject is not explicitly specified, use the one specified in ref
+	if iopts.refObject == "" {
+		refSpec, err := reference.Parse(ref)
+		if err != nil {
+			return iopts, err
+		}
+		iopts.refObject = refSpec.Object
+	}
+	return iopts, nil
+}
+
+// Import imports an image from a Tar stream using reader.
+// OCI format is assumed by default.
+//
+// Note that unreferenced blobs are imported to the content store as well.
+func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) {
+	iopts, err := resolveImportOpt(ref, opts...)
+	if err != nil {
+		return nil, err
+	}
+	switch iopts.format {
+	case ociImageFormat:
+		return c.importFromOCITar(ctx, ref, reader, iopts)
+	default:
+		return nil, errors.Errorf("unsupported format: %s", iopts.format)
+	}
+}
+
+type exportOpts struct {
+	format imageFormat
+}
+
+type ExportOpt func(c *exportOpts) error
+
+func WithOCIExportFormat() ExportOpt {
+	return func(c *exportOpts) error {
+		if c.format != "" {
+			return errors.New("format already set")
+		}
+		c.format = ociImageFormat
+		return nil
+	}
+}
+
+// TODO: add WithMediaTypeTranslation that transforms media types according to the format.
+// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip
+//      -> application/vnd.oci.image.layer.v1.tar+gzip
+
+// Export exports an image to a Tar stream.
+// OCI format is used by default.
+// It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc.
+func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) {
+	var eopts exportOpts
+	for _, o := range opts {
+		if err := o(&eopts); err != nil {
+			return nil, err
+		}
+	}
+	// use OCI as the default format
+	if eopts.format == "" {
+		eopts.format = ociImageFormat
+	}
+	pr, pw := io.Pipe()
+	switch eopts.format {
+	case ociImageFormat:
+		go func() {
+			pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts))
+		}()
+	default:
+		return nil, errors.Errorf("unsupported format: %s", eopts.format)
+	}
+	return pr, nil
+}

cmd/ctr/export.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"io"
+	"os"
+
+	"github.com/containerd/containerd/reference"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli"
+)
+
+var imagesExportCommand = cli.Command{
+	Name:      "export",
+	Usage:     "export an image",
+	ArgsUsage: "[flags] <out> <image>",
+	Description: `Export an image to a tar stream
+`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "oci-ref-name",
+			Value: "",
+			Usage: "Override org.opencontainers.image.ref.name annotation",
+		},
+		cli.StringFlag{
+			Name:  "manifest",
+			Usage: "Digest of manifest",
+		},
+		cli.StringFlag{
+			Name:  "manifest-type",
+			Usage: "Media type of manifest digest",
+			Value: ocispec.MediaTypeImageManifest,
+		},
+	},
+	Action: func(clicontext *cli.Context) error {
+		var (
+			out   = clicontext.Args().First()
+			local = clicontext.Args().Get(1)
+			desc  ocispec.Descriptor
+		)
+
+		ctx, cancel := appContext(clicontext)
+		defer cancel()
+
+		client, err := newClient(clicontext)
+		if err != nil {
+			return err
+		}
+
+		if manifest := clicontext.String("manifest"); manifest != "" {
+			desc.Digest, err = digest.Parse(manifest)
+			if err != nil {
+				return errors.Wrap(err, "invalid manifest digest")
+			}
+			desc.MediaType = clicontext.String("manifest-type")
+		} else {
+			img, err := client.ImageService().Get(ctx, local)
+			if err != nil {
+				return errors.Wrap(err, "unable to resolve image to manifest")
+			}
+			desc = img.Target
+		}
+
+		if desc.Annotations == nil {
+			desc.Annotations = make(map[string]string)
+		}
+		if s, ok := desc.Annotations[ocispec.AnnotationRefName]; !ok || s == "" {
+			if ociRefName := determineOCIRefName(local); ociRefName != "" {
+				desc.Annotations[ocispec.AnnotationRefName] = ociRefName
+			}
+			if ociRefName := clicontext.String("oci-ref-name"); ociRefName != "" {
+				desc.Annotations[ocispec.AnnotationRefName] = ociRefName
+			}
+		}
+		var w io.WriteCloser
+		if out == "-" {
+			w = os.Stdout
+		} else {
+			w, err = os.Create(out)
+			if err != nil {
+				return nil
+			}
+		}
+		r, err := client.Export(ctx, desc)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(w, r); err != nil {
+			return err
+		}
+		if err := w.Close(); err != nil {
+			return err
+		}
+		return r.Close()
+	},
+}
+
+func determineOCIRefName(local string) string {
+	refspec, err := reference.Parse(local)
+	if err != nil {
+		return ""
+	}
+	tag, _ := reference.SplitObject(refspec.Object)
+	return tag
+}

cmd/ctr/images.go

@@ -22,6 +22,8 @@ var imageCommand = cli.Command{
 		imagesListCommand,
 		imageRemoveCommand,
 		imagesSetLabelsCommand,
+		imagesImportCommand,
+		imagesExportCommand,
 	},
 }
 

cmd/ctr/import.go

@@ -0,0 +1,70 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/log"
+	"github.com/urfave/cli"
+)
+
+var imagesImportCommand = cli.Command{
+	Name:      "import",
+	Usage:     "import an image",
+	ArgsUsage: "[flags] <ref> <in>",
+	Description: `Import an image from a tar stream
+`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "ref-object",
+			Value: "",
+			Usage: "reference object e.g. tag@digest (default: use the object specified in ref)",
+		},
+	},
+	Action: func(clicontext *cli.Context) error {
+		var (
+			ref       = clicontext.Args().First()
+			in        = clicontext.Args().Get(1)
+			refObject = clicontext.String("ref-object")
+		)
+
+		ctx, cancel := appContext(clicontext)
+		defer cancel()
+
+		client, err := newClient(clicontext)
+		if err != nil {
+			return err
+		}
+
+		var r io.ReadCloser
+		if in == "-" {
+			r = os.Stdin
+		} else {
+			r, err = os.Open(in)
+			if err != nil {
+				return err
+			}
+		}
+		img, err := client.Import(ctx,
+			ref,
+			r,
+			containerd.WithRefObject(refObject),
+		)
+		if err != nil {
+			return err
+		}
+		if err = r.Close(); err != nil {
+			return err
+		}
+
+		log.G(ctx).WithField("image", ref).Debug("unpacking")
+
+		// TODO: Show unpack status
+		fmt.Printf("unpacking %s...", img.Target().Digest)
+		err = img.Unpack(ctx, clicontext.String("snapshotter"))
+		fmt.Println("done")
+		return err
+	},
+}

content/content.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"time"
 
+	"github.com/containerd/containerd/oci"
 	"github.com/opencontainers/go-digest"
 )
 
@@ -77,10 +78,8 @@ type IngestManager interface {
 }
 
 type Writer interface {
-	io.WriteCloser
+	oci.BlobWriter
 	Status() (Status, error)
-	Digest() digest.Digest
-	Commit(size int64, expected digest.Digest) error
 	Truncate(size int64) error
 }
 

content/local/writer.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/oci"
 	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
 )
@@ -78,7 +79,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error {
 	}
 
 	if size > 0 && size != fi.Size() {
-		return errors.Errorf("%q failed size validation: %v != %v", w.ref, fi.Size(), size)
+		return oci.ErrUnexpectedSize{Expected: size, Actual: fi.Size()}
 	}
 
 	if err := w.fp.Close(); err != nil {
@@ -87,7 +88,7 @@ func (w *writer) Commit(size int64, expected digest.Digest) error {
 
 	dgst := w.digester.Digest()
 	if expected != "" && expected != dgst {
-		return errors.Errorf("unexpected digest: %v != %v", dgst, expected)
+		return oci.ErrUnexpectedDigest{Expected: expected, Actual: dgst}
 	}
 
 	var (

export.go

@@ -0,0 +1,65 @@
+package containerd
+
+import (
+	"archive/tar"
+	"context"
+	"io"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/oci"
+	ocispecs "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error {
+	tw := tar.NewWriter(writer)
+	img := oci.Tar(tw)
+
+	// For tar, we defer creating index until end of the function.
+	if err := oci.Init(img, oci.InitOpts{SkipCreateIndex: true}); err != nil {
+		return err
+	}
+	cs := c.ContentStore()
+	handlers := images.Handlers(
+		images.ChildrenHandler(cs),
+		exportHandler(cs, img),
+	)
+	// For tar, we need to use Walk instead of Dispatch for ensuring sequential write
+	if err := images.Walk(ctx, handlers, desc); err != nil {
+		return err
+	}
+	// For tar, we don't use oci.PutManifestDescriptorToIndex() which allows appending desc to existing index.json
+	// but requires img to support random read access so as to read index.json.
+	return oci.WriteIndex(img,
+		ocispec.Index{
+			Versioned: ocispecs.Versioned{
+				SchemaVersion: 2,
+			},
+			Manifests: []ocispec.Descriptor{desc},
+		},
+	)
+}
+
+func exportHandler(cs content.Store, img oci.ImageDriver) images.HandlerFunc {
+	return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
+		r, err := cs.Reader(ctx, desc.Digest)
+		if err != nil {
+			return nil, err
+		}
+		w, err := oci.NewBlobWriter(img, desc.Digest.Algorithm())
+		if err != nil {
+			return nil, err
+		}
+		if _, err = io.Copy(w, r); err != nil {
+			return nil, err
+		}
+		if err = w.Commit(desc.Size, desc.Digest); err != nil {
+			return nil, err
+		}
+		if err = w.Close(); err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+}

export_test.go

@@ -0,0 +1,64 @@
+package containerd
+
+import (
+	"archive/tar"
+	"io"
+	"runtime"
+	"testing"
+)
+
+// TestExport exports testImage as a tar stream
+func TestExport(t *testing.T) {
+	// TODO: support windows
+	if testing.Short() || runtime.GOOS == "windows" {
+		t.Skip()
+	}
+	ctx, cancel := testContext()
+	defer cancel()
+
+	client, err := New(address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	pulled, err := client.Pull(ctx, testImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exportedStream, err := client.Export(ctx, pulled.Target())
+	if err != nil {
+		t.Fatal(err)
+	}
+	assertOCITar(t, exportedStream)
+}
+
+func assertOCITar(t *testing.T, r io.Reader) {
+	// TODO: add more assertion
+	tr := tar.NewReader(r)
+	foundOCILayout := false
+	foundIndexJSON := false
+	for {
+		h, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+		if h.Name == "oci-layout" {
+			foundOCILayout = true
+		}
+		if h.Name == "index.json" {
+			foundIndexJSON = true
+		}
+	}
+	if !foundOCILayout {
+		t.Error("oci-layout not found")
+	}
+	if !foundIndexJSON {
+		t.Error("index.json not found")
+	}
+}

import.go

@@ -0,0 +1,119 @@
+package containerd
+
+import (
+	"archive/tar"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"strings"
+
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/reference"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) {
+	tag, dgst := reference.SplitObject(refObject)
+	if tag == "" && dgst == "" {
+		return nil, errors.Errorf("unexpected object: %q", refObject)
+	}
+	for _, m := range idx.Manifests {
+		if m.Digest == dgst {
+			return &m, nil
+		}
+		annot, ok := m.Annotations[ocispec.AnnotationRefName]
+		if ok && annot == tag && tag != "" {
+			return &m, nil
+		}
+	}
+	return nil, errors.Errorf("not found: %q", refObject)
+}
+
+func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) {
+	tr := tar.NewReader(reader)
+	store := c.ContentStore()
+	var desc *ocispec.Descriptor
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
+			continue
+		}
+		if hdr.Name == "index.json" {
+			desc, err = onUntarIndexJSON(tr, iopts.refObject)
+			if err != nil {
+				return nil, err
+			}
+			continue
+		}
+		if strings.HasPrefix(hdr.Name, "blobs/") {
+			if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil {
+				return nil, err
+			}
+		}
+	}
+	if desc == nil {
+		return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject)
+	}
+	imgrec := images.Image{
+		Name:   ref,
+		Target: *desc,
+	}
+	is := c.ImageService()
+	if updated, err := is.Update(ctx, imgrec, "target"); err != nil {
+		if !errdefs.IsNotFound(err) {
+			return nil, err
+		}
+
+		created, err := is.Create(ctx, imgrec)
+		if err != nil {
+			return nil, err
+		}
+
+		imgrec = created
+	} else {
+		imgrec = updated
+	}
+
+	img := &image{
+		client: c,
+		i:      imgrec,
+	}
+	return img, nil
+}
+
+func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) {
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	var idx ocispec.Index
+	if err := json.Unmarshal(b, &idx); err != nil {
+		return nil, err
+	}
+	return resolveOCIIndex(idx, refObject)
+}
+
+func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error {
+	// name is like "blobs/sha256/deadbeef"
+	split := strings.Split(name, "/")
+	if len(split) != 3 {
+		return errors.Errorf("unexpected name: %q", name)
+	}
+	algo := digest.Algorithm(split[1])
+	if !algo.Available() {
+		return errors.Errorf("unsupported algorithm: %s", algo)
+	}
+	dgst := digest.NewDigestFromHex(algo.String(), split[2])
+	return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst)
+}

import_test.go

@@ -0,0 +1,44 @@
+package containerd
+
+import (
+	"runtime"
+	"testing"
+)
+
+// TestExportAndImport exports testImage as a tar stream,
+// and import the tar stream as a new image.
+func TestExportAndImport(t *testing.T) {
+	// TODO: support windows
+	if testing.Short() || runtime.GOOS == "windows" {
+		t.Skip()
+	}
+	ctx, cancel := testContext()
+	defer cancel()
+
+	client, err := New(address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	pulled, err := client.Pull(ctx, testImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	exported, err := client.Export(ctx, pulled.Target())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	importRef := "test/export-and-import:tmp"
+	_, err = client.Import(ctx, importRef, exported, WithRefObject("@"+pulled.Target().Digest.String()))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = client.ImageService().Delete(ctx, importRef)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

oci/oci.go

@@ -1,9 +1,12 @@
 // Package oci provides basic operations for manipulating OCI images.
+// This package can be used even outside of containerd, and contains some
+// functions not used in containerd itself.
 package oci
 
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
@@ -14,53 +17,107 @@ import (
 	spec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
-// Init initializes the img directory as an OCI image.
+// BlobWriter writes an OCI blob and returns a digest when committed.
-// i.e. Creates oci-layout, index.json, and blobs.
+type BlobWriter interface {
-//
+	// Close is expected to be called after Commit() when commission is needed.
-// img directory must not exist before calling this function.
+	io.WriteCloser
-//
+	// Digest may return empty digest or panics until committed.
+	Digest() digest.Digest
+	// Commit commits the blob (but no roll-back is guaranteed on an error).
+	// size and expected can be zero-value when unknown.
+	Commit(size int64, expected digest.Digest) error
+}
+
+// ErrUnexpectedSize can be returned from BlobWriter.Commit()
+type ErrUnexpectedSize struct {
+	Expected int64
+	Actual   int64
+}
+
+func (e ErrUnexpectedSize) Error() string {
+	if e.Expected > 0 && e.Expected != e.Actual {
+		return fmt.Sprintf("unexpected size: %d != %d", e.Expected, e.Actual)
+	}
+	return fmt.Sprintf("malformed ErrUnexpectedSize(%+v)", e)
+}
+
+// ErrUnexpectedDigest can be returned from BlobWriter.Commit()
+type ErrUnexpectedDigest struct {
+	Expected digest.Digest
+	Actual   digest.Digest
+}
+
+func (e ErrUnexpectedDigest) Error() string {
+	if e.Expected.String() != "" && e.Expected.String() != e.Actual.String() {
+		return fmt.Sprintf("unexpected digest: %v != %v", e.Expected, e.Actual)
+	}
+	return fmt.Sprintf("malformed ErrUnexpectedDigest(%+v)", e)
+}
+
+// ImageDriver corresponds to the representation of an image.
+// Path uses os.PathSeparator as the separator.
+// The methods of ImageDriver should only be called from oci package.
+type ImageDriver interface {
+	Init() error
+	Remove(path string) error
+	Reader(path string) (io.ReadCloser, error)
+	Writer(path string, perm os.FileMode) (io.WriteCloser, error)
+	BlobWriter(algo digest.Algorithm) (BlobWriter, error)
+}
+
+type InitOpts struct {
 	// imageLayoutVersion can be an empty string for specifying the default version.
-func Init(img, imageLayoutVersion string) error {
+	ImageLayoutVersion string
+	// skip creating oci-layout
+	SkipCreateImageLayout bool
+	// skip creating index.json
+	SkipCreateIndex bool
+}
+
+// Init initializes an OCI image structure.
+// Init calls img.Init, creates `oci-layout`(0444), and creates `index.json`(0644).
+//
+func Init(img ImageDriver, opts InitOpts) error {
+	if err := img.Init(); err != nil {
+		return err
+	}
+
+	// Create oci-layout
+	if !opts.SkipCreateImageLayout {
+		imageLayoutVersion := opts.ImageLayoutVersion
 		if imageLayoutVersion == "" {
 			imageLayoutVersion = spec.ImageLayoutVersion
 		}
-	if _, err := os.Stat(img); err == nil {
-		return os.ErrExist
-	}
-	// Create the directory
-	if err := os.MkdirAll(img, 0755); err != nil {
-		return err
-	}
-	// Create blobs/sha256
-	if err := os.MkdirAll(
-		filepath.Join(img, "blobs", string(digest.Canonical)),
-		0755); err != nil {
-		return nil
-	}
-	// Create oci-layout
 		if err := WriteImageLayout(img, spec.ImageLayout{Version: imageLayoutVersion}); err != nil {
 			return err
 		}
+	}
+
 	// Create index.json
-	return WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
+	if !opts.SkipCreateIndex {
+		if err := WriteIndex(img, spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}}); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
-func blobPath(img string, d digest.Digest) string {
+func blobPath(d digest.Digest) string {
-	return filepath.Join(img, "blobs", d.Algorithm().String(), d.Hex())
+	return filepath.Join("blobs", d.Algorithm().String(), d.Hex())
 }
 
-func indexPath(img string) string {
+const (
-	return filepath.Join(img, "index.json")
+	indexPath = "index.json"
-}
+)
 
 // GetBlobReader returns io.ReadCloser for a blob.
-func GetBlobReader(img string, d digest.Digest) (io.ReadCloser, error) {
+func GetBlobReader(img ImageDriver, d digest.Digest) (io.ReadCloser, error) {
 	// we return a reader rather than the full *os.File here so as to prohibit write operations.
-	return os.Open(blobPath(img, d))
+	return img.Reader(blobPath(d))
 }
 
 // ReadBlob reads an OCI blob.
-func ReadBlob(img string, d digest.Digest) ([]byte, error) {
+func ReadBlob(img ImageDriver, d digest.Digest) ([]byte, error) {
 	r, err := GetBlobReader(img, d)
 	if err != nil {
 		return nil, err
@@ -71,90 +128,47 @@ func ReadBlob(img string, d digest.Digest) ([]byte, error) {
 
 // WriteBlob writes bytes as an OCI blob and returns its digest using the canonical digest algorithm.
 // If you need to specify certain algorithm, you can use NewBlobWriter(img string, algo digest.Algorithm).
-func WriteBlob(img string, b []byte) (digest.Digest, error) {
+func WriteBlob(img ImageDriver, b []byte) (digest.Digest, error) {
-	d := digest.FromBytes(b)
+	w, err := img.BlobWriter(digest.Canonical)
-	return d, ioutil.WriteFile(blobPath(img, d), b, 0444)
+	if err != nil {
+		return "", err
 	}
-
+	n, err := w.Write(b)
-// BlobWriter writes an OCI blob and returns a digest when closed.
+	if err != nil {
-type BlobWriter interface {
+		return "", err
-	io.Writer
-	io.Closer
-	// Digest returns the digest when closed.
-	// Digest panics when the writer is not closed.
-	Digest() digest.Digest
 	}
-
+	if n < len(b) {
-// blobWriter implements BlobWriter.
+		return "", io.ErrShortWrite
-type blobWriter struct {
+	}
-	img      string
+	if err := w.Close(); err != nil {
-	digester digest.Digester
+		return "", err
-	f        *os.File
+	}
-	closed   bool
+	return w.Digest(), err
 }
 
 // NewBlobWriter returns a BlobWriter.
-func NewBlobWriter(img string, algo digest.Algorithm) (BlobWriter, error) {
+func NewBlobWriter(img ImageDriver, algo digest.Algorithm) (BlobWriter, error) {
-	// use img rather than the default tmp, so as to make sure rename(2) can be applied
+	return img.BlobWriter(algo)
-	f, err := ioutil.TempFile(img, "tmp.blobwriter")
-	if err != nil {
-		return nil, err
-	}
-	return &blobWriter{
-		img:      img,
-		digester: algo.Digester(),
-		f:        f,
-	}, nil
-}
-
-// Write implements io.Writer.
-func (bw *blobWriter) Write(b []byte) (int, error) {
-	n, err := bw.f.Write(b)
-	if err != nil {
-		return n, err
-	}
-	return bw.digester.Hash().Write(b)
-}
-
-// Close implements io.Closer.
-func (bw *blobWriter) Close() error {
-	oldPath := bw.f.Name()
-	if err := bw.f.Close(); err != nil {
-		return err
-	}
-	newPath := blobPath(bw.img, bw.digester.Digest())
-	if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil {
-		return err
-	}
-	if err := os.Chmod(oldPath, 0444); err != nil {
-		return err
-	}
-	if err := os.Rename(oldPath, newPath); err != nil {
-		return err
-	}
-	bw.closed = true
-	return nil
-}
-
-// Digest returns the digest when closed.
-func (bw *blobWriter) Digest() digest.Digest {
-	if !bw.closed {
-		panic("blobWriter is unclosed")
-	}
-	return bw.digester.Digest()
 }
 
 // DeleteBlob deletes an OCI blob.
-func DeleteBlob(img string, d digest.Digest) error {
+func DeleteBlob(img ImageDriver, d digest.Digest) error {
-	return os.Remove(blobPath(img, d))
+	return img.Remove(blobPath(d))
 }
 
 // ReadImageLayout returns the image layout.
-func ReadImageLayout(img string) (spec.ImageLayout, error) {
+func ReadImageLayout(img ImageDriver) (spec.ImageLayout, error) {
-	b, err := ioutil.ReadFile(filepath.Join(img, spec.ImageLayoutFile))
+	r, err := img.Reader(spec.ImageLayoutFile)
 	if err != nil {
 		return spec.ImageLayout{}, err
 	}
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return spec.ImageLayout{}, err
+	}
+	if err := r.Close(); err != nil {
+		return spec.ImageLayout{}, err
+	}
 	var layout spec.ImageLayout
 	if err := json.Unmarshal(b, &layout); err != nil {
 		return spec.ImageLayout{}, err
@@ -163,20 +177,38 @@ func ReadImageLayout(img string) (spec.ImageLayout, error) {
 }
 
 // WriteImageLayout writes the image layout.
-func WriteImageLayout(img string, layout spec.ImageLayout) error {
+func WriteImageLayout(img ImageDriver, layout spec.ImageLayout) error {
 	b, err := json.Marshal(layout)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(filepath.Join(img, spec.ImageLayoutFile), b, 0644)
+	w, err := img.Writer(spec.ImageLayoutFile, 0444)
+	if err != nil {
+		return err
+	}
+	n, err := w.Write(b)
+	if err != nil {
+		return err
+	}
+	if n < len(b) {
+		return io.ErrShortWrite
+	}
+	return w.Close()
 }
 
 // ReadIndex returns the index.
-func ReadIndex(img string) (spec.Index, error) {
+func ReadIndex(img ImageDriver) (spec.Index, error) {
-	b, err := ioutil.ReadFile(indexPath(img))
+	r, err := img.Reader(indexPath)
 	if err != nil {
 		return spec.Index{}, err
 	}
+	b, err := ioutil.ReadAll(r)
+	if err != nil {
+		return spec.Index{}, err
+	}
+	if err := r.Close(); err != nil {
+		return spec.Index{}, err
+	}
 	var idx spec.Index
 	if err := json.Unmarshal(b, &idx); err != nil {
 		return spec.Index{}, err
@@ -185,17 +217,28 @@ func ReadIndex(img string) (spec.Index, error) {
 }
 
 // WriteIndex writes the index.
-func WriteIndex(img string, idx spec.Index) error {
+func WriteIndex(img ImageDriver, idx spec.Index) error {
 	b, err := json.Marshal(idx)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(indexPath(img), b, 0644)
+	w, err := img.Writer(indexPath, 0644)
+	if err != nil {
+		return err
+	}
+	n, err := w.Write(b)
+	if err != nil {
+		return err
+	}
+	if n < len(b) {
+		return io.ErrShortWrite
+	}
+	return w.Close()
 }
 
 // RemoveManifestDescriptorFromIndex removes the manifest descriptor from the index.
 // Returns nil error when the entry not found.
-func RemoveManifestDescriptorFromIndex(img string, refName string) error {
+func RemoveManifestDescriptorFromIndex(img ImageDriver, refName string) error {
 	if refName == "" {
 		return errors.New("empty refName specified")
 	}
@@ -217,7 +260,7 @@ func RemoveManifestDescriptorFromIndex(img string, refName string) error {
 
 // PutManifestDescriptorToIndex puts a manifest descriptor to the index.
 // If ref name is set and conflicts with the existing descriptors, the old ones are removed.
-func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
+func PutManifestDescriptorToIndex(img ImageDriver, desc spec.Descriptor) error {
 	refName, ok := desc.Annotations[spec.AnnotationRefName]
 	if ok && refName != "" {
 		if err := RemoveManifestDescriptorFromIndex(img, refName); err != nil {
@@ -233,7 +276,7 @@ func PutManifestDescriptorToIndex(img string, desc spec.Descriptor) error {
 }
 
 // WriteJSONBlob is an utility function that writes x as a JSON blob with the specified media type, and returns the descriptor.
-func WriteJSONBlob(img string, x interface{}, mediaType string) (spec.Descriptor, error) {
+func WriteJSONBlob(img ImageDriver, x interface{}, mediaType string) (spec.Descriptor, error) {
 	b, err := json.Marshal(x)
 	if err != nil {
 		return spec.Descriptor{}, err

oci/oci_test.go

@@ -1,168 +0,0 @@
-package oci
-
-import (
-	"encoding/json"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/containerd/containerd/fs/fstest"
-	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/image-spec/specs-go"
-	spec "github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestInitError(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "oci")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tmp)
-	err = Init(tmp, "")
-	assert.Error(t, err, "file exists")
-}
-
-func TestInit(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "oci")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tmp)
-	img := filepath.Join(tmp, "foo")
-	err = Init(img, "")
-	assert.Nil(t, err)
-	ociLayout, err := json.Marshal(spec.ImageLayout{Version: spec.ImageLayoutVersion})
-	assert.Nil(t, err)
-	indexJSON, err := json.Marshal(spec.Index{Versioned: specs.Versioned{SchemaVersion: 2}})
-	applier := fstest.Apply(
-		fstest.CreateDir("/foo", 0755),
-		fstest.CreateDir("/foo/blobs", 0755),
-		fstest.CreateDir("/foo/blobs/"+string(digest.Canonical), 0755),
-		fstest.CreateFile("/foo/oci-layout", ociLayout, 0644),
-		fstest.CreateFile("/foo/index.json", indexJSON, 0644),
-	)
-	err = fstest.CheckDirectoryEqualWithApplier(tmp, applier)
-	assert.Nil(t, err)
-}
-
-func TestWriteReadDeleteBlob(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "oci")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tmp)
-	img := filepath.Join(tmp, "foo")
-	err = Init(img, "")
-	assert.Nil(t, err)
-	testBlob := []byte("test")
-	// Write
-	d, err := WriteBlob(img, testBlob)
-	applier := fstest.Apply(
-		fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
-	)
-	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
-	assert.Nil(t, err)
-	// Read
-	b, err := ReadBlob(img, d)
-	assert.Nil(t, err)
-	assert.Equal(t, testBlob, b)
-	// Delete
-	err = DeleteBlob(img, d)
-	assert.Nil(t, err)
-	applier = fstest.Apply()
-	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
-	assert.Nil(t, err)
-}
-
-func TestBlobWriter(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "oci")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tmp)
-	img := filepath.Join(tmp, "foo")
-	err = Init(img, "")
-	assert.Nil(t, err)
-	testBlob := []byte("test")
-	w, err := NewBlobWriter(img, digest.Canonical)
-	_, err = w.Write(testBlob)
-	assert.Nil(t, err)
-	// blob is not written until closing
-	applier := fstest.Apply()
-	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
-	// digest is unavailable until closing
-	assert.Panics(t, func() { w.Digest() })
-	// close and calculate the digest
-	err = w.Close()
-	assert.Nil(t, err)
-	d := w.Digest()
-	applier = fstest.Apply(
-		fstest.CreateFile("/"+d.Hex(), testBlob, 0444),
-	)
-	err = fstest.CheckDirectoryEqualWithApplier(filepath.Join(img, "blobs", string(digest.Canonical)), applier)
-	assert.Nil(t, err)
-}
-
-func TestIndex(t *testing.T) {
-	tmp, err := ioutil.TempDir("", "oci")
-	assert.Nil(t, err)
-	defer os.RemoveAll(tmp)
-	img := filepath.Join(tmp, "foo")
-	err = Init(img, "")
-	assert.Nil(t, err)
-	descs := []spec.Descriptor{
-		{
-			MediaType: spec.MediaTypeImageManifest,
-			Annotations: map[string]string{
-				spec.AnnotationRefName: "foo",
-				"dummy":                "desc0",
-			},
-		},
-		{
-			MediaType: spec.MediaTypeImageManifest,
-			Annotations: map[string]string{
-				// will be removed later
-				spec.AnnotationRefName: "bar",
-				"dummy":                "desc1",
-			},
-		},
-		{
-			MediaType: spec.MediaTypeImageManifest,
-			Annotations: map[string]string{
-				// duplicated ref name
-				spec.AnnotationRefName: "foo",
-				"dummy":                "desc2",
-			},
-		},
-		{
-			MediaType: spec.MediaTypeImageManifest,
-			Annotations: map[string]string{
-				// no ref name
-				"dummy": "desc3",
-			},
-		},
-	}
-	for _, desc := range descs {
-		err := PutManifestDescriptorToIndex(img, desc)
-		assert.Nil(t, err)
-	}
-	err = RemoveManifestDescriptorFromIndex(img, "bar")
-	assert.Nil(t, err)
-	expected := spec.Index{
-		Versioned: specs.Versioned{SchemaVersion: 2},
-		Manifests: []spec.Descriptor{
-			{
-				MediaType: spec.MediaTypeImageManifest,
-				Annotations: map[string]string{
-					// duplicated ref name
-					spec.AnnotationRefName: "foo",
-					"dummy":                "desc2",
-				},
-			},
-			{
-				MediaType: spec.MediaTypeImageManifest,
-				Annotations: map[string]string{
-					// no ref name
-					"dummy": "desc3",
-				},
-			},
-		},
-	}
-	idx, err := ReadIndex(img)
-	assert.Nil(t, err)
-	assert.Equal(t, expected, idx)
-}

oci/tar.go

@@ -0,0 +1,156 @@
+package oci
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/opencontainers/go-digest"
+)
+
+// TarWriter is an interface that is implemented by archive/tar.Writer.
+// (Using an interface allows hooking)
+type TarWriter interface {
+	io.WriteCloser
+	Flush() error
+	WriteHeader(hdr *tar.Header) error
+}
+
+// Tar is ImageDriver for TAR representation of an OCI image.
+func Tar(w TarWriter) ImageDriver {
+	return &tarDriver{
+		w: w,
+	}
+}
+
+type tarDriver struct {
+	w TarWriter
+}
+
+func (d *tarDriver) Init() error {
+	headers := []tar.Header{
+		{
+			Name:     "blobs/",
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		},
+		{
+			Name:     "blobs/" + string(digest.Canonical) + "/",
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		},
+	}
+	for _, h := range headers {
+		if err := d.w.WriteHeader(&h); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (d *tarDriver) Remove(path string) error {
+	return errors.New("Tar does not support Remove")
+}
+
+func (d *tarDriver) Reader(path string) (io.ReadCloser, error) {
+	// because tar does not support random access
+	return nil, errors.New("Tar does not support Reader")
+}
+
+func (d *tarDriver) Writer(path string, perm os.FileMode) (io.WriteCloser, error) {
+	name := filepath.ToSlash(path)
+	return &tarDriverWriter{
+		w:    d.w,
+		name: name,
+		mode: int64(perm),
+	}, nil
+}
+
+// tarDriverWriter is used for writing non-blob files
+// (e.g. oci-layout, index.json)
+type tarDriverWriter struct {
+	bytes.Buffer
+	w    TarWriter
+	name string
+	mode int64
+}
+
+func (w *tarDriverWriter) Close() error {
+	if err := w.w.WriteHeader(&tar.Header{
+		Name:     w.name,
+		Mode:     w.mode,
+		Size:     int64(w.Len()),
+		Typeflag: tar.TypeReg,
+	}); err != nil {
+		return err
+	}
+	n, err := io.Copy(w.w, w)
+	if err != nil {
+		return err
+	}
+	if n < int64(w.Len()) {
+		return io.ErrShortWrite
+	}
+	return w.w.Flush()
+}
+
+func (d *tarDriver) BlobWriter(algo digest.Algorithm) (BlobWriter, error) {
+	return &tarBlobWriter{
+		w:        d.w,
+		digester: algo.Digester(),
+	}, nil
+}
+
+// tarBlobWriter implements BlobWriter.
+type tarBlobWriter struct {
+	w        TarWriter
+	digester digest.Digester
+	buf      bytes.Buffer // TODO: use tmp file for large buffer?
+}
+
+// Write implements io.Writer.
+func (bw *tarBlobWriter) Write(b []byte) (int, error) {
+	n, err := bw.buf.Write(b)
+	if err != nil {
+		return n, err
+	}
+	return bw.digester.Hash().Write(b)
+}
+
+func (bw *tarBlobWriter) Commit(size int64, expected digest.Digest) error {
+	path := "blobs/" + bw.digester.Digest().Algorithm().String() + "/" + bw.digester.Digest().Hex()
+	if err := bw.w.WriteHeader(&tar.Header{
+		Name:     path,
+		Mode:     0444,
+		Size:     int64(bw.buf.Len()),
+		Typeflag: tar.TypeReg,
+	}); err != nil {
+		return err
+	}
+	n, err := io.Copy(bw.w, &bw.buf)
+	if err != nil {
+		return err
+	}
+	if n < int64(bw.buf.Len()) {
+		return io.ErrShortWrite
+	}
+	if size > 0 && size != n {
+		return ErrUnexpectedSize{Expected: size, Actual: n}
+	}
+	if expected != "" && bw.digester.Digest() != expected {
+		return ErrUnexpectedDigest{Expected: expected, Actual: bw.digester.Digest()}
+	}
+	return bw.w.Flush()
+}
+
+func (bw *tarBlobWriter) Close() error {
+	// we don't close bw.w (reused for writing another blob)
+	return bw.w.Flush()
+}
+
+func (bw *tarBlobWriter) Digest() digest.Digest {
+	return bw.digester.Digest()
+}