Refactor differ into separate package

Add differ options and package with interface.
Update optional values on diff interface to use options.

Signed-off-by: Derek McGowan <derek@mcgstyle.net>

diff/diff.go

@@ -0,0 +1,69 @@
+package diff
+
+import (
+	"github.com/containerd/containerd/mount"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"golang.org/x/net/context"
+)
+
+// Config is used to hold parameters needed for a diff operation
+type Config struct {
+	// MediaType is the type of diff to generate
+	// Default depends on the differ,
+	// i.e. application/vnd.oci.image.layer.v1.tar+gzip
+	MediaType string
+
+	// Reference is the content upload reference
+	// Default will use a random reference string
+	Reference string
+
+	// Labels are the labels to apply to the generated content
+	Labels map[string]string
+}
+
+// Opt is used to configure a diff operation
+type Opt func(*Config) error
+
+// Differ allows the apply and creation of filesystem diffs between mounts
+type Differ interface {
+	// Apply applies the content referred to by the given descriptor to
+	// the provided mount. The method of applying is based on the
+	// implementation and content descriptor. For example, in the common
+	// case the descriptor is a file system difference in tar format,
+	// that tar would be applied on top of the mounts.
+	Apply(ctx context.Context, desc ocispec.Descriptor, mount []mount.Mount) (ocispec.Descriptor, error)
+
+	// DiffMounts computes the difference between two mounts and returns a
+	// descriptor for the computed diff. The options can provide
+	// a ref which can be used to track the content creation of the diff.
+	// The media type which is used to determine the format of the created
+	// content can also be provided as an option.
+	DiffMounts(ctx context.Context, lower, upper []mount.Mount, opts ...Opt) (ocispec.Descriptor, error)
+}
+
+// WithMediaType sets the media type to use for creating the diff, without
+// specifying the differ will choose a default.
+func WithMediaType(m string) Opt {
+	return func(c *Config) error {
+		c.MediaType = m
+		return nil
+	}
+}
+
+// WithReference is used to set the content upload reference used by
+// the diff operation. This allows the caller to track the upload through
+// the content store.
+func WithReference(ref string) Opt {
+	return func(c *Config) error {
+		c.Reference = ref
+		return nil
+	}
+}
+
+// WithLabels is used to set content labels on the created diff content.
+func WithLabels(labels map[string]string) Opt {
+	return func(c *Config) error {
+		c.Labels = labels
+		return nil
+	}
+}

diff/walking/differ.go

@@ -0,0 +1,245 @@
+package walking
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/archive"
+	"github.com/containerd/containerd/archive/compression"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/plugin"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type: plugin.DiffPlugin,
+		ID:   "walking",
+		Requires: []plugin.Type{
+			plugin.MetadataPlugin,
+		},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			md, err := ic.Get(plugin.MetadataPlugin)
+			if err != nil {
+				return nil, err
+			}
+
+			ic.Meta.Platforms = append(ic.Meta.Platforms, platforms.DefaultSpec())
+			return NewWalkingDiff(md.(*metadata.DB).ContentStore())
+		},
+	})
+}
+
+type walkingDiff struct {
+	store content.Store
+}
+
+var emptyDesc = ocispec.Descriptor{}
+
+// NewWalkingDiff is a generic implementation of diff.Differ.
+// NewWalkingDiff is expected to work with any filesystem.
+func NewWalkingDiff(store content.Store) (diff.Differ, error) {
+	return &walkingDiff{
+		store: store,
+	}, nil
+}
+
+// Apply applies the content associated with the provided digests onto the
+// provided mounts. Archive content will be extracted and decompressed if
+// necessary.
+func (s *walkingDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts []mount.Mount) (ocispec.Descriptor, error) {
+	var isCompressed bool
+	switch desc.MediaType {
+	case ocispec.MediaTypeImageLayer, images.MediaTypeDockerSchema2Layer:
+	case ocispec.MediaTypeImageLayerGzip, images.MediaTypeDockerSchema2LayerGzip:
+		isCompressed = true
+	default:
+		// Still apply all generic media types *.tar[.+]gzip and *.tar
+		if strings.HasSuffix(desc.MediaType, ".tar.gzip") || strings.HasSuffix(desc.MediaType, ".tar+gzip") {
+			isCompressed = true
+		} else if !strings.HasSuffix(desc.MediaType, ".tar") {
+			return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", desc.MediaType)
+		}
+	}
+
+	dir, err := ioutil.TempDir("", "extract-")
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to create temporary directory")
+	}
+	defer os.RemoveAll(dir)
+
+	if err := mount.All(mounts, dir); err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to mount")
+	}
+	defer mount.Unmount(dir, 0)
+
+	ra, err := s.store.ReaderAt(ctx, desc.Digest)
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to get reader from content store")
+	}
+	defer ra.Close()
+
+	r := content.NewReader(ra)
+	if isCompressed {
+		ds, err := compression.DecompressStream(r)
+		if err != nil {
+			return emptyDesc, err
+		}
+		defer ds.Close()
+		r = ds
+	}
+
+	digester := digest.Canonical.Digester()
+	rc := &readCounter{
+		r: io.TeeReader(r, digester.Hash()),
+	}
+
+	if _, err := archive.Apply(ctx, dir, rc); err != nil {
+		return emptyDesc, err
+	}
+
+	// Read any trailing data
+	if _, err := io.Copy(ioutil.Discard, rc); err != nil {
+		return emptyDesc, err
+	}
+
+	return ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageLayer,
+		Size:      rc.c,
+		Digest:    digester.Digest(),
+	}, nil
+}
+
+// DiffMounts creates a diff between the given mounts and uploads the result
+// to the content store.
+func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount, opts ...diff.Opt) (ocispec.Descriptor, error) {
+	var config diff.Config
+	for _, opt := range opts {
+		if err := opt(&config); err != nil {
+			return emptyDesc, err
+		}
+	}
+
+	if config.MediaType == "" {
+		config.MediaType = ocispec.MediaTypeImageLayerGzip
+	}
+
+	var isCompressed bool
+	switch config.MediaType {
+	case ocispec.MediaTypeImageLayer:
+	case ocispec.MediaTypeImageLayerGzip:
+		isCompressed = true
+	default:
+		return emptyDesc, errors.Wrapf(errdefs.ErrNotImplemented, "unsupported diff media type: %v", config.MediaType)
+	}
+	aDir, err := ioutil.TempDir("", "left-")
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to create temporary directory")
+	}
+	defer os.RemoveAll(aDir)
+
+	bDir, err := ioutil.TempDir("", "right-")
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to create temporary directory")
+	}
+	defer os.RemoveAll(bDir)
+
+	if err := mount.All(lower, aDir); err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to mount")
+	}
+	defer mount.Unmount(aDir, 0)
+
+	if err := mount.All(upper, bDir); err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to mount")
+	}
+	defer mount.Unmount(bDir, 0)
+
+	if config.Reference == "" {
+		config.Reference = uniqueRef()
+	}
+
+	cw, err := s.store.Writer(ctx, config.Reference, 0, "")
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to open writer")
+	}
+
+	if isCompressed {
+		dgstr := digest.SHA256.Digester()
+		compressed, err := compression.CompressStream(cw, compression.Gzip)
+		if err != nil {
+			cw.Close()
+			return emptyDesc, errors.Wrap(err, "failed to get compressed stream")
+		}
+		err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash()), aDir, bDir)
+		compressed.Close()
+		if err != nil {
+			cw.Close()
+			return emptyDesc, errors.Wrap(err, "failed to write compressed diff")
+		}
+
+		if config.Labels == nil {
+			config.Labels = map[string]string{}
+		}
+		config.Labels["containerd.io/uncompressed"] = dgstr.Digest().String()
+	} else {
+		if err = archive.WriteDiff(ctx, cw, aDir, bDir); err != nil {
+			cw.Close()
+			return emptyDesc, errors.Wrap(err, "failed to write diff")
+		}
+	}
+
+	var commitopts []content.Opt
+	if config.Labels != nil {
+		commitopts = append(commitopts, content.WithLabels(config.Labels))
+	}
+
+	dgst := cw.Digest()
+	if err := cw.Commit(ctx, 0, dgst, commitopts...); err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to commit")
+	}
+
+	info, err := s.store.Info(ctx, dgst)
+	if err != nil {
+		return emptyDesc, errors.Wrap(err, "failed to get info from content store")
+	}
+
+	return ocispec.Descriptor{
+		MediaType: config.MediaType,
+		Size:      info.Size,
+		Digest:    info.Digest,
+	}, nil
+}
+
+type readCounter struct {
+	r io.Reader
+	c int64
+}
+
+func (rc *readCounter) Read(p []byte) (n int, err error) {
+	n, err = rc.r.Read(p)
+	rc.c += int64(n)
+	return
+}
+
+func uniqueRef() string {
+	t := time.Now()
+	var b [3]byte
+	// Ignore read failures, just decreases uniqueness
+	rand.Read(b[:])
+	return fmt.Sprintf("%d-%s", t.UnixNano(), base64.URLEncoding.EncodeToString(b[:]))
+}