Merge pull request #2632 from ehazlett/checkpoint-restore

Refactor checkpoint and restore to client

client.go

@@ -17,7 +17,9 @@
 package containerd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"net/http"
 	"runtime"
@@ -520,6 +522,45 @@ func (c *Client) ListImages(ctx context.Context, filters ...string) ([]Image, er
 	return images, nil
 }
 
+// Restore restores a container from a checkpoint
+func (c *Client) Restore(ctx context.Context, id string, checkpoint Image, opts ...RestoreOpts) (Container, error) {
+	store := c.ContentStore()
+	index, err := decodeIndex(ctx, store, checkpoint.Target())
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, done, err := c.WithLease(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer done(ctx)
+
+	copts := []NewContainerOpts{}
+	for _, o := range opts {
+		copts = append(copts, o(ctx, id, c, checkpoint, index))
+	}
+
+	ctr, err := c.NewContainer(ctx, id, copts...)
+	if err != nil {
+		return nil, err
+	}
+
+	return ctr, nil
+}
+
+func writeIndex(ctx context.Context, index *ocispec.Index, client *Client, ref string) (d ocispec.Descriptor, err error) {
+	labels := map[string]string{}
+	for i, m := range index.Manifests {
+		labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = m.Digest.String()
+	}
+	data, err := json.Marshal(index)
+	if err != nil {
+		return ocispec.Descriptor{}, err
+	}
+	return writeContent(ctx, client.ContentStore(), ocispec.MediaTypeImageIndex, ref, bytes.NewReader(data), content.WithLabels(labels))
+}
+
 // Subscribe to events that match one or more of the provided filters.
 //
 // Callers should listen on both the envelope and errs channels. If the errs

cmd/ctr/commands/commands.go

@@ -71,10 +71,6 @@ var (
 			Name:  "config,c",
 			Usage: "path to the runtime-specific spec config file",
 		},
-		cli.StringFlag{
-			Name:  "checkpoint",
-			Usage: "provide the checkpoint digest to restore the container",
-		},
 		cli.StringFlag{
 			Name:  "cwd",
 			Usage: "specify the working directory of the process",

cmd/ctr/commands/containers/containers.go

@@ -18,7 +18,6 @@ package containers
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"strings"
@@ -29,8 +28,10 @@ import (
 	"github.com/containerd/containerd/cmd/ctr/commands"
 	"github.com/containerd/containerd/cmd/ctr/commands/run"
 	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 )
 
@@ -45,6 +46,8 @@ var Command = cli.Command{
 		infoCommand,
 		listCommand,
 		setLabelsCommand,
+		checkpointCommand,
+		restoreCommand,
 	},
 }
 
@@ -282,3 +285,152 @@ var infoCommand = cli.Command{
 		return nil
 	},
 }
+
+var checkpointCommand = cli.Command{
+	Name:      "checkpoint",
+	Usage:     "checkpoint a container",
+	ArgsUsage: "CONTAINER REF",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "rw",
+			Usage: "include the rw layer in the checkpoint",
+		},
+		cli.BoolFlag{
+			Name:  "image",
+			Usage: "include the image in the checkpoint",
+		},
+		cli.BoolFlag{
+			Name:  "task",
+			Usage: "checkpoint container task",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		id := context.Args().First()
+		if id == "" {
+			return errors.New("container id must be provided")
+		}
+		ref := context.Args().Get(1)
+		if ref == "" {
+			return errors.New("ref must be provided")
+		}
+		client, ctx, cancel, err := commands.NewClient(context)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+		opts := []containerd.CheckpointOpts{
+			containerd.WithCheckpointRuntime,
+		}
+
+		if context.Bool("image") {
+			opts = append(opts, containerd.WithCheckpointImage)
+		}
+		if context.Bool("rw") {
+			opts = append(opts, containerd.WithCheckpointRW)
+		}
+		if context.Bool("task") {
+			opts = append(opts, containerd.WithCheckpointTask)
+		}
+		container, err := client.LoadContainer(ctx, id)
+		if err != nil {
+			return err
+		}
+		task, err := container.Task(ctx, nil)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				return err
+			}
+		}
+		// pause if running
+		if task != nil {
+			if err := task.Pause(ctx); err != nil {
+				return err
+			}
+			defer func() {
+				if err := task.Resume(ctx); err != nil {
+					fmt.Println(errors.Wrap(err, "error resuming task"))
+				}
+			}()
+		}
+
+		if _, err := container.Checkpoint(ctx, ref, opts...); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}
+
+var restoreCommand = cli.Command{
+	Name:      "restore",
+	Usage:     "restore a container from checkpoint",
+	ArgsUsage: "CONTAINER REF",
+	Flags: []cli.Flag{
+		cli.BoolFlag{
+			Name:  "rw",
+			Usage: "restore the rw layer from the checkpoint",
+		},
+		cli.BoolFlag{
+			Name:  "live",
+			Usage: "restore the runtime and memory data from the checkpoint",
+		},
+	},
+	Action: func(context *cli.Context) error {
+		id := context.Args().First()
+		if id == "" {
+			return errors.New("container id must be provided")
+		}
+		ref := context.Args().Get(1)
+		if ref == "" {
+			return errors.New("ref must be provided")
+		}
+		client, ctx, cancel, err := commands.NewClient(context)
+		if err != nil {
+			return err
+		}
+		defer cancel()
+
+		checkpoint, err := client.GetImage(ctx, ref)
+		if err != nil {
+			if !errdefs.IsNotFound(err) {
+				return err
+			}
+			// TODO (ehazlett): consider other options (always/never fetch)
+			ck, err := client.Fetch(ctx, ref)
+			if err != nil {
+				return err
+			}
+			checkpoint = containerd.NewImage(client, ck)
+		}
+
+		opts := []containerd.RestoreOpts{
+			containerd.WithRestoreImage,
+			containerd.WithRestoreSpec,
+			containerd.WithRestoreRuntime,
+		}
+		if context.Bool("rw") {
+			opts = append(opts, containerd.WithRestoreRW)
+		}
+
+		ctr, err := client.Restore(ctx, id, checkpoint, opts...)
+		if err != nil {
+			return err
+		}
+
+		topts := []containerd.NewTaskOpts{}
+		if context.Bool("live") {
+			topts = append(topts, containerd.WithTaskCheckpoint(checkpoint))
+		}
+
+		task, err := ctr.NewTask(ctx, cio.NewCreator(cio.WithStdio), topts...)
+		if err != nil {
+			return err
+		}
+
+		if err := task.Start(ctx); err != nil {
+			return err
+		}
+
+		return nil
+	},
+}

cmd/ctr/commands/run/run_unix.go

@@ -44,14 +44,6 @@ func NewContainer(ctx gocontext.Context, client *containerd.Client, context *cli
 		id = context.Args().Get(1)
 	}
 
-	if raw := context.String("checkpoint"); raw != "" {
-		im, err := client.GetImage(ctx, raw)
-		if err != nil {
-			return nil, err
-		}
-		return client.NewContainer(ctx, id, containerd.WithCheckpoint(im, id), containerd.WithRuntime(context.String("runtime"), nil))
-	}
-
 	var (
 		opts  []oci.SpecOpts
 		cOpts []containerd.NewContainerOpts

container.go

@@ -28,12 +28,22 @@ import (
 	"github.com/containerd/containerd/cio"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/typeurl"
 	prototypes "github.com/gogo/protobuf/types"
+	ver "github.com/opencontainers/image-spec/specs-go"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
+const (
+	checkpointImageNameLabel       = "org.opencontainers.image.ref.name"
+	checkpointRuntimeNameLabel     = "io.containerd.checkpoint.runtime"
+	checkpointSnapshotterNameLabel = "io.containerd.checkpoint.snapshotter"
+)
+
 // Container is a metadata object for container resources and task creation
 type Container interface {
 	// ID identifies the container
@@ -64,6 +74,8 @@ type Container interface {
 	Extensions(context.Context) (map[string]prototypes.Any, error)
 	// Update a container
 	Update(context.Context, ...UpdateContainerOpts) error
+	// Checkpoint creates a checkpoint image of the current container
+	Checkpoint(context.Context, string, ...CheckpointOpts) (Image, error)
 }
 
 func containerFromRecord(client *Client, c containers.Container) *container {
@@ -272,6 +284,70 @@ func (c *container) Update(ctx context.Context, opts ...UpdateContainerOpts) err
 	return nil
 }
 
+func (c *container) Checkpoint(ctx context.Context, ref string, opts ...CheckpointOpts) (Image, error) {
+	index := &ocispec.Index{
+		Versioned: ver.Versioned{
+			SchemaVersion: 2,
+		},
+		Annotations: make(map[string]string),
+	}
+	copts := &options.CheckpointOptions{
+		Exit:                false,
+		OpenTcp:             false,
+		ExternalUnixSockets: false,
+		Terminal:            false,
+		FileLocks:           true,
+		EmptyNamespaces:     nil,
+	}
+	info, err := c.Info(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	img, err := c.Image(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, done, err := c.client.WithLease(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer done(ctx)
+
+	// add image name to manifest
+	index.Annotations[checkpointImageNameLabel] = img.Name()
+	// add runtime info to index
+	index.Annotations[checkpointRuntimeNameLabel] = info.Runtime.Name
+	// add snapshotter info to index
+	index.Annotations[checkpointSnapshotterNameLabel] = info.Snapshotter
+
+	// process remaining opts
+	for _, o := range opts {
+		if err := o(ctx, c.client, &info, index, copts); err != nil {
+			err = errdefs.FromGRPC(err)
+			if !errdefs.IsAlreadyExists(err) {
+				return nil, err
+			}
+		}
+	}
+
+	desc, err := writeIndex(ctx, index, c.client, c.ID()+"index")
+	if err != nil {
+		return nil, err
+	}
+	i := images.Image{
+		Name:   ref,
+		Target: desc,
+	}
+	checkpoint, err := c.client.ImageService().Create(ctx, i)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewImage(c.client, checkpoint), nil
+}
+
 func (c *container) loadTask(ctx context.Context, ioAttach cio.Attach) (Task, error) {
 	response, err := c.client.TaskService().Get(ctx, &tasks.GetRequest{
 		ContainerID: c.id,

container_checkpoint_opts.go

@@ -0,0 +1,155 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"runtime"
+
+	tasks "github.com/containerd/containerd/api/services/tasks/v1"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/diff"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/rootfs"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/typeurl"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+var (
+	// ErrCheckpointRWUnsupported is returned if the container runtime does not support checkpoint
+	ErrCheckpointRWUnsupported = errors.New("rw checkpoint is only supported on v2 runtimes")
+	// ErrMediaTypeNotFound returns an error when a media type in the manifest is unknown
+	ErrMediaTypeNotFound = errors.New("media type not found")
+)
+
+// CheckpointOpts are options to manage the checkpoint operation
+type CheckpointOpts func(context.Context, *Client, *containers.Container, *imagespec.Index, *options.CheckpointOptions) error
+
+// WithCheckpointImage includes the container image in the checkpoint
+func WithCheckpointImage(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error {
+	ir, err := client.ImageService().Get(ctx, c.Image)
+	if err != nil {
+		return err
+	}
+	index.Manifests = append(index.Manifests, ir.Target)
+	return nil
+}
+
+// WithCheckpointTask includes the running task
+func WithCheckpointTask(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error {
+	any, err := typeurl.MarshalAny(copts)
+	if err != nil {
+		return nil
+	}
+	task, err := client.TaskService().Checkpoint(ctx, &tasks.CheckpointTaskRequest{
+		ContainerID: c.ID,
+		Options:     any,
+	})
+	if err != nil {
+		return err
+	}
+	for _, d := range task.Descriptors {
+		platformSpec := platforms.DefaultSpec()
+		index.Manifests = append(index.Manifests, imagespec.Descriptor{
+			MediaType: d.MediaType,
+			Size:      d.Size_,
+			Digest:    d.Digest,
+			Platform:  &platformSpec,
+		})
+	}
+	// save copts
+	data, err := any.Marshal()
+	if err != nil {
+		return err
+	}
+	r := bytes.NewReader(data)
+	desc, err := writeContent(ctx, client.ContentStore(), images.MediaTypeContainerd1CheckpointOptions, c.ID+"-checkpoint-options", r)
+	if err != nil {
+		return err
+	}
+	desc.Platform = &imagespec.Platform{
+		OS:           runtime.GOOS,
+		Architecture: runtime.GOARCH,
+	}
+	index.Manifests = append(index.Manifests, desc)
+	return nil
+}
+
+// WithCheckpointRuntime includes the container runtime info
+func WithCheckpointRuntime(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error {
+	if c.Runtime.Options != nil {
+		data, err := c.Runtime.Options.Marshal()
+		if err != nil {
+			return err
+		}
+		r := bytes.NewReader(data)
+		desc, err := writeContent(ctx, client.ContentStore(), images.MediaTypeContainerd1CheckpointRuntimeOptions, c.ID+"-runtime-options", r)
+		if err != nil {
+			return err
+		}
+		desc.Platform = &imagespec.Platform{
+			OS:           runtime.GOOS,
+			Architecture: runtime.GOARCH,
+		}
+		index.Manifests = append(index.Manifests, desc)
+	}
+	return nil
+}
+
+// WithCheckpointRW includes the rw in the checkpoint
+func WithCheckpointRW(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error {
+	diffOpts := []diff.Opt{
+		diff.WithReference(fmt.Sprintf("checkpoint-rw-%s", c.SnapshotKey)),
+	}
+	rw, err := rootfs.CreateDiff(ctx,
+		c.SnapshotKey,
+		client.SnapshotService(c.Snapshotter),
+		client.DiffService(),
+		diffOpts...,
+	)
+	if err != nil {
+		return err
+
+	}
+	rw.Platform = &imagespec.Platform{
+		OS:           runtime.GOOS,
+		Architecture: runtime.GOARCH,
+	}
+	index.Manifests = append(index.Manifests, rw)
+	return nil
+}
+
+// WithCheckpointTaskExit causes the task to exit after checkpoint
+func WithCheckpointTaskExit(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error {
+	copts.Exit = true
+	return nil
+}
+
+// GetIndexByMediaType returns the index in a manifest for the specified media type
+func GetIndexByMediaType(index *imagespec.Index, mt string) (*imagespec.Descriptor, error) {
+	for _, d := range index.Manifests {
+		if d.MediaType == mt {
+			return &d, nil
+		}
+	}
+	return nil, ErrMediaTypeNotFound
+}

container_checkpoint_test.go

@@ -28,8 +28,11 @@ import (
 	"testing"
 
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/runtime/linux/runctypes"
+)
-	"github.com/containerd/containerd/runtime/v2/runc/options"
+
+const (
+	v1runtime          = "io.containerd.runtime.v1.linux"
+	testCheckpointName = "checkpoint-test:latest"
 )
 
 func TestCheckpointRestorePTY(t *testing.T) {
@@ -41,6 +44,9 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.Close()
+	if client.runtime == v1runtime {
+		t.Skip()
+	}
 
 	var (
 		ctx, cancel = testContext()
@@ -56,7 +62,8 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		WithNewSnapshot(id, image),
 		WithNewSpec(oci.WithImageConfig(image),
 			oci.WithProcessArgs("sh", "-c", "read A; echo z${A}z"),
-			oci.WithTTY))
+			oci.WithTTY),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -83,7 +90,12 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, withExit(client))
+	checkpoint, err := container.Checkpoint(ctx, testCheckpointName+"withpty", []CheckpointOpts{
+		WithCheckpointRuntime,
+		WithCheckpointRW,
+		WithCheckpointTaskExit,
+		WithCheckpointTask,
+	}...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -94,6 +106,10 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		t.Fatal(err)
 	}
 	direct.Delete()
+	if err := container.Delete(ctx, WithSnapshotCleanup); err != nil {
+		t.Fatal(err)
+	}
+
 	direct, err = newDirectIO(ctx, true)
 	if err != nil {
 		t.Fatal(err)
@@ -109,6 +125,14 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		io.Copy(buf, direct.Stdout)
 	}()
 
+	if container, err = client.Restore(ctx, id, checkpoint, []RestoreOpts{
+		WithRestoreImage,
+		WithRestoreSpec,
+		WithRestoreRuntime,
+		WithRestoreRW,
+	}...); err != nil {
+		t.Fatal(err)
+	}
 	if task, err = container.NewTask(ctx, direct.IOCreate,
 		WithTaskCheckpoint(checkpoint)); err != nil {
 		t.Fatal(err)
@@ -146,6 +170,9 @@ func TestCheckpointRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.Close()
+	if client.runtime == v1runtime {
+		t.Skip()
+	}
 
 	var (
 		ctx, cancel = testContext()
@@ -157,7 +184,7 @@ func TestCheckpointRestore(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "100")))
+	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "10")))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +205,11 @@ func TestCheckpointRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, withExit(client))
+	checkpoint, err := container.Checkpoint(ctx, testCheckpointName+"restore", []CheckpointOpts{
+		WithCheckpointRuntime,
+		WithCheckpointRW,
+		WithCheckpointTask,
+	}...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -188,6 +219,18 @@ func TestCheckpointRestore(t *testing.T) {
 	if _, err := task.Delete(ctx); err != nil {
 		t.Fatal(err)
 	}
+	if err := container.Delete(ctx, WithSnapshotCleanup); err != nil {
+		t.Fatal(err)
+	}
+
+	if container, err = client.Restore(ctx, id, checkpoint, []RestoreOpts{
+		WithRestoreImage,
+		WithRestoreSpec,
+		WithRestoreRuntime,
+		WithRestoreRW,
+	}...); err != nil {
+		t.Fatal(err)
+	}
 	if task, err = container.NewTask(ctx, empty(), WithTaskCheckpoint(checkpoint)); err != nil {
 		t.Fatal(err)
 	}
@@ -217,6 +260,9 @@ func TestCheckpointRestoreNewContainer(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.Close()
+	if client.runtime == v1runtime {
+		t.Skip()
+	}
 
 	id := t.Name()
 	ctx, cancel := testContext()
@@ -226,7 +272,7 @@ func TestCheckpointRestoreNewContainer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "100")))
+	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "5")))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -247,7 +293,11 @@ func TestCheckpointRestoreNewContainer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, withExit(client))
+	checkpoint, err := container.Checkpoint(ctx, testCheckpointName+"newcontainer", []CheckpointOpts{
+		WithCheckpointRuntime,
+		WithCheckpointRW,
+		WithCheckpointTask,
+	}...)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -260,7 +310,12 @@ func TestCheckpointRestoreNewContainer(t *testing.T) {
 	if err := container.Delete(ctx, WithSnapshotCleanup); err != nil {
 		t.Fatal(err)
 	}
-	if container, err = client.NewContainer(ctx, id, WithCheckpoint(checkpoint, id)); err != nil {
+	if container, err = client.Restore(ctx, id, checkpoint, []RestoreOpts{
+		WithRestoreImage,
+		WithRestoreSpec,
+		WithRestoreRuntime,
+		WithRestoreRW,
+	}...); err != nil {
 		t.Fatal(err)
 	}
 	if task, err = container.NewTask(ctx, empty(), WithTaskCheckpoint(checkpoint)); err != nil {
@@ -290,11 +345,14 @@ func TestCheckpointLeaveRunning(t *testing.T) {
 	if !supportsCriu {
 		t.Skip("system does not have criu installed")
 	}
-	client, err := New(address)
+	client, err := newClient(t, address)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer client.Close()
+	if client.runtime == v1runtime {
+		t.Skip()
+	}
 
 	var (
 		ctx, cancel = testContext()
@@ -327,7 +385,12 @@ func TestCheckpointLeaveRunning(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := task.Checkpoint(ctx); err != nil {
+	// checkpoint
+	if _, err := container.Checkpoint(ctx, testCheckpointName+"leaverunning", []CheckpointOpts{
+		WithCheckpointRuntime,
+		WithCheckpointRW,
+		WithCheckpointTask,
+	}...); err != nil {
 		t.Fatal(err)
 	}
 
@@ -345,19 +408,3 @@ func TestCheckpointLeaveRunning(t *testing.T) {
 
 	<-statusC
 }
-
-func withExit(client *Client) CheckpointTaskOpts {
-	return func(r *CheckpointTaskInfo) error {
-		switch client.runtime {
-		case "io.containerd.runc.v1":
-			r.Options = &options.CheckpointOptions{
-				Exit: true,
-			}
-		default:
-			r.Options = &runctypes.CheckpointOptions{
-				Exit: true,
-			}
-		}
-		return nil
-	}
-}

container_opts_unix.go

@@ -26,81 +26,12 @@ import (
 	"syscall"
 
 	"github.com/containerd/containerd/containers"
-	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
-	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/platforms"
-	"github.com/gogo/protobuf/proto"
-	protobuf "github.com/gogo/protobuf/types"
 	"github.com/opencontainers/image-spec/identity"
-	"github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/pkg/errors"
 )
 
-// WithCheckpoint allows a container to be created from the checkpointed information
-// provided by the descriptor. The image, snapshot, and runtime specifications are
-// restored on the container
-func WithCheckpoint(im Image, snapshotKey string) NewContainerOpts {
-	// set image and rw, and spec
-	return func(ctx context.Context, client *Client, c *containers.Container) error {
-		var (
-			desc  = im.Target()
-			store = client.ContentStore()
-		)
-		index, err := decodeIndex(ctx, store, desc)
-		if err != nil {
-			return err
-		}
-		var rw *v1.Descriptor
-		for _, m := range index.Manifests {
-			switch m.MediaType {
-			case v1.MediaTypeImageLayer:
-				fk := m
-				rw = &fk
-			case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema2ManifestList:
-				config, err := images.Config(ctx, store, m, platforms.Default())
-				if err != nil {
-					return errors.Wrap(err, "unable to resolve image config")
-				}
-				diffIDs, err := images.RootFS(ctx, store, config)
-				if err != nil {
-					return errors.Wrap(err, "unable to get rootfs")
-				}
-				setSnapshotterIfEmpty(c)
-				if _, err := client.SnapshotService(c.Snapshotter).Prepare(ctx, snapshotKey, identity.ChainID(diffIDs).String()); err != nil {
-					if !errdefs.IsAlreadyExists(err) {
-						return err
-					}
-				}
-				c.Image = index.Annotations["image.name"]
-			case images.MediaTypeContainerd1CheckpointConfig:
-				data, err := content.ReadBlob(ctx, store, m)
-				if err != nil {
-					return errors.Wrap(err, "unable to read checkpoint config")
-				}
-				var any protobuf.Any
-				if err := proto.Unmarshal(data, &any); err != nil {
-					return err
-				}
-				c.Spec = &any
-			}
-		}
-		if rw != nil {
-			// apply the rw snapshot to the new rw layer
-			mounts, err := client.SnapshotService(c.Snapshotter).Mounts(ctx, snapshotKey)
-			if err != nil {
-				return errors.Wrapf(err, "unable to get mounts for %s", snapshotKey)
-			}
-			if _, err := client.DiffService().Apply(ctx, *rw, mounts); err != nil {
-				return errors.Wrap(err, "unable to apply rw diff")
-			}
-		}
-		c.SnapshotKey = snapshotKey
-		return nil
-	}
-}
-
 // WithRemappedSnapshot creates a new snapshot and remaps the uid/gid for the
 // filesystem to be used by a container with user namespaces
 func WithRemappedSnapshot(id string, i Image, uid, gid uint32) NewContainerOpts {

container_restore_opts.go

@@ -0,0 +1,150 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package containerd
+
+import (
+	"context"
+
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/platforms"
+	"github.com/gogo/protobuf/proto"
+	ptypes "github.com/gogo/protobuf/types"
+	"github.com/opencontainers/image-spec/identity"
+	imagespec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+var (
+	// ErrImageNameNotFoundInIndex is returned when the image name is not found in the index
+	ErrImageNameNotFoundInIndex = errors.New("image name not found in index")
+	// ErrRuntimeNameNotFoundInIndex is returned when the runtime is not found in the index
+	ErrRuntimeNameNotFoundInIndex = errors.New("runtime not found in index")
+	// ErrSnapshotterNameNotFoundInIndex is returned when the snapshotter is not found in the index
+	ErrSnapshotterNameNotFoundInIndex = errors.New("snapshotter not found in index")
+)
+
+// RestoreOpts are options to manage the restore operation
+type RestoreOpts func(context.Context, string, *Client, Image, *imagespec.Index) NewContainerOpts
+
+// WithRestoreImage restores the image for the container
+func WithRestoreImage(ctx context.Context, id string, client *Client, checkpoint Image, index *imagespec.Index) NewContainerOpts {
+	return func(ctx context.Context, client *Client, c *containers.Container) error {
+		name, ok := index.Annotations[checkpointImageNameLabel]
+		if !ok || name == "" {
+			return ErrRuntimeNameNotFoundInIndex
+		}
+		snapshotter, ok := index.Annotations[checkpointSnapshotterNameLabel]
+		if !ok || name == "" {
+			return ErrSnapshotterNameNotFoundInIndex
+		}
+		i, err := client.GetImage(ctx, name)
+		if err != nil {
+			return err
+		}
+
+		diffIDs, err := i.(*image).i.RootFS(ctx, client.ContentStore(), platforms.Default())
+		if err != nil {
+			return err
+		}
+		parent := identity.ChainID(diffIDs).String()
+		if _, err := client.SnapshotService(snapshotter).Prepare(ctx, id, parent); err != nil {
+			return err
+		}
+		c.Image = i.Name()
+		c.SnapshotKey = id
+		c.Snapshotter = snapshotter
+		return nil
+	}
+}
+
+// WithRestoreRuntime restores the runtime for the container
+func WithRestoreRuntime(ctx context.Context, id string, client *Client, checkpoint Image, index *imagespec.Index) NewContainerOpts {
+	return func(ctx context.Context, client *Client, c *containers.Container) error {
+		name, ok := index.Annotations[checkpointRuntimeNameLabel]
+		if !ok {
+			return ErrRuntimeNameNotFoundInIndex
+		}
+
+		// restore options if present
+		m, err := GetIndexByMediaType(index, images.MediaTypeContainerd1CheckpointRuntimeOptions)
+		if err != nil {
+			if err != ErrMediaTypeNotFound {
+				return err
+			}
+		}
+		var options *ptypes.Any
+		if m != nil {
+			store := client.ContentStore()
+			data, err := content.ReadBlob(ctx, store, *m)
+			if err != nil {
+				return errors.Wrap(err, "unable to read checkpoint runtime")
+			}
+			if err := proto.Unmarshal(data, options); err != nil {
+				return err
+			}
+		}
+
+		c.Runtime = containers.RuntimeInfo{
+			Name:    name,
+			Options: options,
+		}
+		return nil
+	}
+}
+
+// WithRestoreSpec restores the spec from the checkpoint for the container
+func WithRestoreSpec(ctx context.Context, id string, client *Client, checkpoint Image, index *imagespec.Index) NewContainerOpts {
+	return func(ctx context.Context, client *Client, c *containers.Container) error {
+		m, err := GetIndexByMediaType(index, images.MediaTypeContainerd1CheckpointConfig)
+		if err != nil {
+			return err
+		}
+		store := client.ContentStore()
+		data, err := content.ReadBlob(ctx, store, *m)
+		if err != nil {
+			return errors.Wrap(err, "unable to read checkpoint config")
+		}
+		var any ptypes.Any
+		if err := proto.Unmarshal(data, &any); err != nil {
+			return err
+		}
+		c.Spec = &any
+		return nil
+	}
+}
+
+// WithRestoreRW restores the rw layer from the checkpoint for the container
+func WithRestoreRW(ctx context.Context, id string, client *Client, checkpoint Image, index *imagespec.Index) NewContainerOpts {
+	return func(ctx context.Context, client *Client, c *containers.Container) error {
+		// apply rw layer
+		rw, err := GetIndexByMediaType(index, imagespec.MediaTypeImageLayerGzip)
+		if err != nil {
+			return err
+		}
+		mounts, err := client.SnapshotService(c.Snapshotter).Mounts(ctx, c.SnapshotKey)
+		if err != nil {
+			return err
+		}
+
+		if _, err := client.DiffService().Apply(ctx, *rw, mounts); err != nil {
+			return err
+		}
+		return nil
+	}
+}

images/mediatypes.go

@@ -29,11 +29,14 @@ const (
 	MediaTypeDockerSchema2Manifest         = "application/vnd.docker.distribution.manifest.v2+json"
 	MediaTypeDockerSchema2ManifestList     = "application/vnd.docker.distribution.manifest.list.v2+json"
 	// Checkpoint/Restore Media Types
-	MediaTypeContainerd1Checkpoint        = "application/vnd.containerd.container.criu.checkpoint.criu.tar"
+	MediaTypeContainerd1Checkpoint               = "application/vnd.containerd.container.criu.checkpoint.criu.tar"
-	MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar"
+	MediaTypeContainerd1CheckpointPreDump        = "application/vnd.containerd.container.criu.checkpoint.predump.tar"
-	MediaTypeContainerd1Resource          = "application/vnd.containerd.container.resource.tar"
+	MediaTypeContainerd1Resource                 = "application/vnd.containerd.container.resource.tar"
-	MediaTypeContainerd1RW                = "application/vnd.containerd.container.rw.tar"
+	MediaTypeContainerd1RW                       = "application/vnd.containerd.container.rw.tar"
-	MediaTypeContainerd1CheckpointConfig  = "application/vnd.containerd.container.checkpoint.config.v1+proto"
+	MediaTypeContainerd1CheckpointConfig         = "application/vnd.containerd.container.checkpoint.config.v1+proto"
+	MediaTypeContainerd1CheckpointOptions        = "application/vnd.containerd.container.checkpoint.options.v1+proto"
+	MediaTypeContainerd1CheckpointRuntimeName    = "application/vnd.containerd.container.checkpoint.runtime.name"
+	MediaTypeContainerd1CheckpointRuntimeOptions = "application/vnd.containerd.container.checkpoint.runtime.options+proto"
 	// Legacy Docker schema1 manifest
 	MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws"
 )