diff --git a/archive/tar.go b/archive/tar.go index b63bbe70f..e98b3fb57 100644 --- a/archive/tar.go +++ b/archive/tar.go @@ -40,13 +40,7 @@ func Diff(ctx context.Context, a, b string) io.ReadCloser { r, w := io.Pipe() go func() { - var err error - cw := newChangeWriter(w, b) - if err = fs.Changes(ctx, a, b, cw.HandleChange); err != nil { - err = errors.Wrap(err, "failed to create diff tar stream") - } else { - err = cw.Close() - } + err := WriteDiff(ctx, w, a, b) if err = w.CloseWithError(err); err != nil { log.G(ctx).WithError(err).Debugf("closing tar pipe failed") } @@ -55,6 +49,22 @@ func Diff(ctx context.Context, a, b string) io.ReadCloser { return r } +// WriteDiff writes a tar stream of the computed difference between the +// provided directories. +// +// Produces a tar using OCI style file markers for deletions. Deleted +// files will be prepended with the prefix ".wh.". This style is +// based off AUFS whiteouts. +// See https://github.com/opencontainers/image-spec/blob/master/layer.md +func WriteDiff(ctx context.Context, w io.Writer, a, b string) error { + cw := newChangeWriter(w, b) + err := fs.Changes(ctx, a, b, cw.HandleChange) + if err != nil { + return errors.Wrap(err, "failed to create diff tar stream") + } + return cw.Close() +} + const ( // whiteoutPrefix prefix means file is a whiteout. If this is followed by a // filename this means that file has been removed from the base layer. diff --git a/cmd/ctr/main.go b/cmd/ctr/main.go index 693ae2afd..bbe673883 100644 --- a/cmd/ctr/main.go +++ b/cmd/ctr/main.go @@ -61,6 +61,7 @@ containerd client execCommand, pauseCommand, resumeCommand, + snapshotCommand, versionCommand, } app.Commands = append(app.Commands, extraCmds...) diff --git a/cmd/ctr/run.go b/cmd/ctr/run.go index 0f9d479a7..5b8908a23 100644 --- a/cmd/ctr/run.go +++ b/cmd/ctr/run.go @@ -54,6 +54,10 @@ var runCommand = cli.Command{ Name: "net-host", Usage: "enable host networking for the container", }, + cli.BoolFlag{ + Name: "keep", + Usage: "keep container after running", + }, }, Action: func(context *cli.Context) error { var ( @@ -110,7 +114,11 @@ var runCommand = cli.Command{ return err } - mounts, err = snapshotter.Prepare(ctx, id, identity.ChainID(diffIDs).String()) + if context.Bool("readonly") { + mounts, err = snapshotter.View(ctx, id, identity.ChainID(diffIDs).String()) + } else { + mounts, err = snapshotter.Prepare(ctx, id, identity.ChainID(diffIDs).String()) + } if err != nil { if !snapshot.IsExist(err) { return err @@ -192,10 +200,12 @@ var runCommand = cli.Command{ if err != nil { return err } - if _, err := containers.Delete(ctx, &execution.DeleteRequest{ - ID: response.ID, - }); err != nil { - return err + if !context.Bool("keep") { + if _, err := containers.Delete(ctx, &execution.DeleteRequest{ + ID: response.ID, + }); err != nil { + return err + } } if status != 0 { return cli.NewExitError("", int(status)) diff --git a/cmd/ctr/snapshot.go b/cmd/ctr/snapshot.go new file mode 100644 index 000000000..631f7d914 --- /dev/null +++ b/cmd/ctr/snapshot.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/containerd/containerd/rootfs" + "github.com/urfave/cli" +) + +var snapshotCommand = cli.Command{ + Name: "snapshot", + Usage: "snapshot a container into an archive", + ArgsUsage: "", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "id", + Usage: "id of the container", + }, + }, + Action: func(clicontext *cli.Context) error { + id := clicontext.String("id") + if id == "" { + return errors.New("container id must be provided") + } + + snapshotter, err := getSnapshotter(clicontext) + if err != nil { + return err + } + + differ, err := getDiffService(clicontext) + if err != nil { + return err + } + + contentRef := fmt.Sprintf("diff-%s", id) + + d, err := rootfs.Diff(context.TODO(), id, contentRef, snapshotter, differ) + if err != nil { + return err + } + + // TODO: Track progress + fmt.Printf("%s %s\n", d.MediaType, d.Digest) + + return nil + }, +} diff --git a/cmd/ctr/utils.go b/cmd/ctr/utils.go index 9db0296ea..c99aa2849 100644 --- a/cmd/ctr/utils.go +++ b/cmd/ctr/utils.go @@ -13,6 +13,7 @@ import ( "github.com/Sirupsen/logrus" contentapi "github.com/containerd/containerd/api/services/content" + diffapi "github.com/containerd/containerd/api/services/diff" "github.com/containerd/containerd/api/services/execution" imagesapi "github.com/containerd/containerd/api/services/images" snapshotapi "github.com/containerd/containerd/api/services/snapshot" @@ -21,6 +22,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" contentservice "github.com/containerd/containerd/services/content" + "github.com/containerd/containerd/services/diff" imagesservice "github.com/containerd/containerd/services/images" snapshotservice "github.com/containerd/containerd/services/snapshot" "github.com/containerd/containerd/snapshot" @@ -62,6 +64,14 @@ func getImageStore(clicontext *cli.Context) (images.Store, error) { return imagesservice.NewStoreFromClient(imagesapi.NewImagesClient(conn)), nil } +func getDiffService(context *cli.Context) (diff.DiffService, error) { + conn, err := getGRPCConnection(context) + if err != nil { + return nil, err + } + return diff.NewDiffServiceFromClient(diffapi.NewDiffClient(conn)), nil +} + func getVersionService(context *cli.Context) (versionservice.VersionClient, error) { conn, err := getGRPCConnection(context) if err != nil { diff --git a/cmd/dist/pull.go b/cmd/dist/pull.go index 1b834f37e..b647aca27 100644 --- a/cmd/dist/pull.go +++ b/cmd/dist/pull.go @@ -116,7 +116,7 @@ command. As part of this process, we do the following: log.G(ctx).Fatal(err) } snapshotter := snapshotservice.NewSnapshotterFromClient(snapshotapi.NewSnapshotClient(conn)) - applier := diffservice.NewApplierFromClient(diffapi.NewDiffClient(conn)) + applier := diffservice.NewDiffServiceFromClient(diffapi.NewDiffClient(conn)) log.G(ctx).Info("unpacking rootfs") diff --git a/cmd/dist/rootfs.go b/cmd/dist/rootfs.go index 70a7b5706..d8711959f 100644 --- a/cmd/dist/rootfs.go +++ b/cmd/dist/rootfs.go @@ -65,7 +65,7 @@ var rootfsUnpackCommand = cli.Command{ } snapshotter := snapshotservice.NewSnapshotterFromClient(snapshotapi.NewSnapshotClient(conn)) - applier := diffservice.NewApplierFromClient(diffapi.NewDiffClient(conn)) + applier := diffservice.NewDiffServiceFromClient(diffapi.NewDiffClient(conn)) chainID, err := rootfs.ApplyLayers(ctx, layers, snapshotter, applier) if err != nil { diff --git a/rootfs/diff.go b/rootfs/diff.go new file mode 100644 index 000000000..a23b84f26 --- /dev/null +++ b/rootfs/diff.go @@ -0,0 +1,52 @@ +package rootfs + +import ( + "context" + "fmt" + + "github.com/containerd/containerd" + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/snapshot" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +type MountDiffer interface { + DiffMounts(ctx context.Context, lower, upper []containerd.Mount, media, ref string) (ocispec.Descriptor, error) +} + +type DiffOptions struct { + MountDiffer + content.Store + snapshot.Snapshotter +} + +func Diff(ctx context.Context, snapshotID, contentRef string, sn snapshot.Snapshotter, md MountDiffer) (ocispec.Descriptor, error) { + info, err := sn.Stat(ctx, snapshotID) + if err != nil { + return ocispec.Descriptor{}, err + } + + lowerKey := fmt.Sprintf("%s-parent-view", info.Parent) + lower, err := sn.View(ctx, lowerKey, info.Parent) + if err != nil { + return ocispec.Descriptor{}, err + } + defer sn.Remove(ctx, lowerKey) + + var upper []containerd.Mount + if info.Kind == snapshot.KindActive { + upper, err = sn.Mounts(ctx, snapshotID) + if err != nil { + return ocispec.Descriptor{}, err + } + } else { + upperKey := fmt.Sprintf("%s-view", snapshotID) + upper, err = sn.View(ctx, upperKey, snapshotID) + if err != nil { + return ocispec.Descriptor{}, err + } + defer sn.Remove(ctx, lowerKey) + } + + return md.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, contentRef) +} diff --git a/services/diff/client.go b/services/diff/client.go index a64e0407a..66a920e50 100644 --- a/services/diff/client.go +++ b/services/diff/client.go @@ -11,19 +11,24 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) +type DiffService interface { + rootfs.Applier + rootfs.MountDiffer +} + // NewApplierFromClient returns a new Applier which communicates // over a GRPC connection. -func NewApplierFromClient(client diffapi.DiffClient) rootfs.Applier { - return &remoteApplier{ +func NewDiffServiceFromClient(client diffapi.DiffClient) DiffService { + return &remote{ client: client, } } -type remoteApplier struct { +type remote struct { client diffapi.DiffClient } -func (r *remoteApplier) Apply(ctx context.Context, diff ocispec.Descriptor, mounts []containerd.Mount) (ocispec.Descriptor, error) { +func (r *remote) Apply(ctx context.Context, diff ocispec.Descriptor, mounts []containerd.Mount) (ocispec.Descriptor, error) { req := &diffapi.ApplyRequest{ Diff: fromDescriptor(diff), Mounts: fromMounts(mounts), @@ -35,6 +40,20 @@ func (r *remoteApplier) Apply(ctx context.Context, diff ocispec.Descriptor, moun return toDescriptor(resp.Applied), nil } +func (r *remote) DiffMounts(ctx context.Context, a, b []containerd.Mount, media, ref string) (ocispec.Descriptor, error) { + req := &diffapi.DiffRequest{ + Left: fromMounts(a), + Right: fromMounts(b), + MediaType: media, + Ref: ref, + } + resp, err := r.client.Diff(ctx, req) + if err != nil { + return ocispec.Descriptor{}, err + } + return toDescriptor(resp.Diff), nil +} + func fromDescriptor(d ocispec.Descriptor) *descriptor.Descriptor { return &descriptor.Descriptor{ MediaType: d.MediaType, diff --git a/services/diff/service.go b/services/diff/service.go index 1c1d16197..7e722df4d 100644 --- a/services/diff/service.go +++ b/services/diff/service.go @@ -102,8 +102,73 @@ func (s *service) Apply(ctx context.Context, er *diffapi.ApplyRequest) (*diffapi return resp, nil } -func (s *service) Diff(context.Context, *diffapi.DiffRequest) (*diffapi.DiffResponse, error) { - return nil, errors.New("not implemented") +func (s *service) Diff(ctx context.Context, dr *diffapi.DiffRequest) (*diffapi.DiffResponse, error) { + aMounts := toMounts(dr.Left) + bMounts := toMounts(dr.Right) + + aDir, err := ioutil.TempDir("", "left-") + if err != nil { + return nil, errors.Wrap(err, "failed to create temporary directory") + } + defer os.RemoveAll(aDir) + + bDir, err := ioutil.TempDir("", "right-") + if err != nil { + return nil, errors.Wrap(err, "failed to create temporary directory") + } + defer os.RemoveAll(bDir) + + if err := containerd.MountAll(aMounts, aDir); err != nil { + return nil, errors.Wrap(err, "failed to mount") + } + defer containerd.Unmount(aDir, 0) + + if err := containerd.MountAll(bMounts, bDir); err != nil { + return nil, errors.Wrap(err, "failed to mount") + } + defer containerd.Unmount(bDir, 0) + + cw, err := s.store.Writer(ctx, dr.Ref, 0, "") + if err != nil { + return nil, errors.Wrap(err, "failed to open writer") + } + + // TODO: Validate media type + + // TODO: Support compressed media types (link compressed to uncompressed) + //dgstr := digest.SHA256.Digester() + //wc := &writeCounter{} + //compressed, err := compression.CompressStream(cw, compression.Gzip) + //if err != nil { + // return nil, errors.Wrap(err, "failed to get compressed stream") + //} + //err = archive.WriteDiff(ctx, io.MultiWriter(compressed, dgstr.Hash(), wc), lowerDir, upperDir) + //compressed.Close() + + err = archive.WriteDiff(ctx, cw, aDir, bDir) + if err != nil { + return nil, errors.Wrap(err, "failed to write diff") + } + + dgst := cw.Digest() + if err := cw.Commit(0, dgst); err != nil { + return nil, errors.Wrap(err, "failed to commit") + } + + info, err := s.store.Info(ctx, dgst) + if err != nil { + return nil, errors.Wrap(err, "failed to get info from content store") + } + + desc := ocispec.Descriptor{ + MediaType: dr.MediaType, + Digest: info.Digest, + Size: info.Size, + } + + return &diffapi.DiffResponse{ + Diff: fromDescriptor(desc), + }, nil } type readCounter struct {