From abbec626202c7cafa81daa2d44ae1f0aed7e97f4 Mon Sep 17 00:00:00 2001 From: Jacob Wen Date: Thu, 14 Sep 2017 15:38:48 +0800 Subject: [PATCH 1/5] cmd/ctr: create an image for checkpoint This allows one to manage the checkpoints by using the `ctr image` command. The image is created with label "containerd.io/checkpoint". By default, it is not included in the output of `ctr images ls`. We can list the images by using the following command: $ ctr images ls labels.containerd.\"io/checkpoint\"==true Fixes #1026 Signed-off-by: Jacob Wen --- cmd/ctr/checkpoint.go | 14 ++++++++++++++ cmd/ctr/images.go | 10 ++++++++++ 2 files changed, 24 insertions(+) diff --git a/cmd/ctr/checkpoint.go b/cmd/ctr/checkpoint.go index f34fc83ab..9841a92aa 100644 --- a/cmd/ctr/checkpoint.go +++ b/cmd/ctr/checkpoint.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/containerd/containerd" + "github.com/containerd/containerd/images" "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -48,6 +49,19 @@ var taskCheckpointCommand = cli.Command{ if err != nil { return err } + + labels := map[string]string{ + "containerd.io/checkpoint": "true", + } + img := images.Image{ + Name: checkpoint.Digest.String(), + Target: checkpoint, + Labels: labels, + } + _, err = client.ImageService().Create(ctx, img) + if err != nil { + return err + } fmt.Println(checkpoint.Digest.String()) return nil }, diff --git a/cmd/ctr/images.go b/cmd/ctr/images.go index 6be8b9ccb..c355f0c7e 100644 --- a/cmd/ctr/images.go +++ b/cmd/ctr/images.go @@ -57,6 +57,16 @@ var imagesListCommand = cli.Command{ imageStore := client.ImageService() cs := client.ContentStore() + if len(filters) == 0 { + filters = append(filters, `labels.containerd."io/checkpoint"!=true`) + } else { + for _, f := range filters { + if !strings.Contains(f, `labels.containerd."io/checkpoint"`) { + f += `,labels.containerd."io/checkpoint"!=true` + _ = f // ignore error: ineffectual assignment to f + } + } + } imageList, err := imageStore.List(ctx, filters...) if err != nil { return errors.Wrap(err, "failed to list images") From a19fd6ed6e3f35be6cfc646dd29861265981fae7 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 17 Oct 2017 11:26:28 -0400 Subject: [PATCH 2/5] Add checkpoint media types to handler Signed-off-by: Michael Crosby --- images/image.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/images/image.go b/images/image.go index 8179dc22c..4c78c6cc2 100644 --- a/images/image.go +++ b/images/image.go @@ -315,7 +315,8 @@ func Children(ctx context.Context, provider content.Provider, desc ocispec.Descr MediaTypeDockerSchema2LayerForeign, MediaTypeDockerSchema2LayerForeignGzip, MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig, ocispec.MediaTypeImageLayer, ocispec.MediaTypeImageLayerGzip, - ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip: + ocispec.MediaTypeImageLayerNonDistributable, ocispec.MediaTypeImageLayerNonDistributableGzip, + MediaTypeContainerd1Checkpoint, MediaTypeContainerd1CheckpointConfig: // childless data types. return nil, nil default: From e201be5196f5e3573b33006cf5232eeafe97014f Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 17 Oct 2017 12:05:11 -0400 Subject: [PATCH 3/5] Create checkpointed image in client Allow a user provided name for the checkpoint as well as a default generated name for the checkpoint image. Signed-off-by: Michael Crosby --- cmd/ctr/checkpoint.go | 16 +----------- cmd/ctr/run.go | 11 ++------ cmd/ctr/run_unix.go | 20 ++++++-------- cmd/ctr/run_windows.go | 3 +-- cmd/ctr/start.go | 3 +-- container_opts_unix.go | 12 ++++++--- task.go | 59 +++++++++++++++++++++++++++++++----------- task_opts.go | 8 ++++++ 8 files changed, 73 insertions(+), 59 deletions(-) diff --git a/cmd/ctr/checkpoint.go b/cmd/ctr/checkpoint.go index 9841a92aa..bf39f7851 100644 --- a/cmd/ctr/checkpoint.go +++ b/cmd/ctr/checkpoint.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/containerd/containerd" - "github.com/containerd/containerd/images" "github.com/pkg/errors" "github.com/urfave/cli" ) @@ -49,20 +48,7 @@ var taskCheckpointCommand = cli.Command{ if err != nil { return err } - - labels := map[string]string{ - "containerd.io/checkpoint": "true", - } - img := images.Image{ - Name: checkpoint.Digest.String(), - Target: checkpoint, - Labels: labels, - } - _, err = client.ImageService().Create(ctx, img) - if err != nil { - return err - } - fmt.Println(checkpoint.Digest.String()) + fmt.Println(checkpoint.Name()) return nil }, } diff --git a/cmd/ctr/run.go b/cmd/ctr/run.go index 10f218aa4..50c851bc6 100644 --- a/cmd/ctr/run.go +++ b/cmd/ctr/run.go @@ -9,7 +9,6 @@ import ( "github.com/containerd/console" "github.com/containerd/containerd" "github.com/containerd/containerd/containers" - digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -96,8 +95,7 @@ var runCommand = cli.Command{ }, snapshotterFlags...), Action: func(context *cli.Context) error { var ( - err error - checkpointIndex digest.Digest + err error ctx, cancel = appContext(context) id = context.Args().Get(1) @@ -112,11 +110,6 @@ var runCommand = cli.Command{ if id == "" { return errors.New("container id must be provided") } - if raw := context.String("checkpoint"); raw != "" { - if checkpointIndex, err = digest.Parse(raw); err != nil { - return err - } - } client, err := newClient(context) if err != nil { return err @@ -128,7 +121,7 @@ var runCommand = cli.Command{ if context.Bool("rm") { defer container.Delete(ctx, containerd.WithSnapshotCleanup) } - task, err := newTask(ctx, container, checkpointIndex, tty) + task, err := newTask(ctx, client, container, context.String("checkpoint"), tty) if err != nil { return err } diff --git a/cmd/ctr/run_unix.go b/cmd/ctr/run_unix.go index 32d04dc82..04be5ab8d 100644 --- a/cmd/ctr/run_unix.go +++ b/cmd/ctr/run_unix.go @@ -11,8 +11,6 @@ import ( "github.com/containerd/console" "github.com/containerd/containerd" - digest "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -67,15 +65,11 @@ func newContainer(ctx gocontext.Context, client *containerd.Client, context *cli ) if raw := context.String("checkpoint"); raw != "" { - checkpointIndex, err := digest.Parse(raw) + im, err := client.GetImage(ctx, raw) if err != nil { return nil, err } - if checkpointIndex != "" { - return client.NewContainer(ctx, id, containerd.WithCheckpoint(v1.Descriptor{ - Digest: checkpointIndex, - }, id)) - } + return client.NewContainer(ctx, id, containerd.WithCheckpoint(im, id)) } var ( @@ -120,7 +114,7 @@ func newContainer(ctx gocontext.Context, client *containerd.Client, context *cli return client.NewContainer(ctx, id, cOpts...) } -func newTask(ctx gocontext.Context, container containerd.Container, checkpoint digest.Digest, tty bool) (containerd.Task, error) { +func newTask(ctx gocontext.Context, client *containerd.Client, container containerd.Container, checkpoint string, tty bool) (containerd.Task, error) { if checkpoint == "" { io := containerd.Stdio if tty { @@ -128,7 +122,9 @@ func newTask(ctx gocontext.Context, container containerd.Container, checkpoint d } return container.NewTask(ctx, io) } - return container.NewTask(ctx, containerd.Stdio, containerd.WithTaskCheckpoint(v1.Descriptor{ - Digest: checkpoint, - })) + im, err := client.GetImage(ctx, checkpoint) + if err != nil { + return nil, err + } + return container.NewTask(ctx, containerd.Stdio, containerd.WithTaskCheckpoint(im)) } diff --git a/cmd/ctr/run_windows.go b/cmd/ctr/run_windows.go index c764230fc..8712663b6 100644 --- a/cmd/ctr/run_windows.go +++ b/cmd/ctr/run_windows.go @@ -9,7 +9,6 @@ import ( "github.com/containerd/containerd/containers" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" - digest "github.com/opencontainers/go-digest" specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -118,7 +117,7 @@ func newContainer(ctx gocontext.Context, client *containerd.Client, context *cli ) } -func newTask(ctx gocontext.Context, container containerd.Container, _ digest.Digest, tty bool) (containerd.Task, error) { +func newTask(ctx gocontext.Context, client *containerd.Client, container containerd.Container, _ string, tty bool) (containerd.Task, error) { io := containerd.Stdio if tty { io = containerd.StdioTerminal diff --git a/cmd/ctr/start.go b/cmd/ctr/start.go index 2f8375c2c..087f6c684 100644 --- a/cmd/ctr/start.go +++ b/cmd/ctr/start.go @@ -2,7 +2,6 @@ package main import ( "github.com/containerd/console" - "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -41,7 +40,7 @@ var taskStartCommand = cli.Command{ tty := spec.Process.Terminal - task, err := newTask(ctx, container, digest.Digest(""), tty) + task, err := newTask(ctx, client, container, "", tty) if err != nil { return err } diff --git a/container_opts_unix.go b/container_opts_unix.go index 75283ef83..e2cd7c939 100644 --- a/container_opts_unix.go +++ b/container_opts_unix.go @@ -24,11 +24,14 @@ import ( // WithCheckpoint allows a container to be created from the checkpointed information // provided by the descriptor. The image, snapshot, and runtime specifications are // restored on the container -func WithCheckpoint(desc v1.Descriptor, snapshotKey string) NewContainerOpts { +func WithCheckpoint(im Image, snapshotKey string) NewContainerOpts { // set image and rw, and spec return func(ctx context.Context, client *Client, c *containers.Container) error { - id := desc.Digest - store := client.ContentStore() + var ( + desc = im.Target() + id = desc.Digest + store = client.ContentStore() + ) index, err := decodeIndex(ctx, store, id) if err != nil { return err @@ -85,8 +88,9 @@ func WithCheckpoint(desc v1.Descriptor, snapshotKey string) NewContainerOpts { // WithTaskCheckpoint allows a task to be created with live runtime and memory data from a // previous checkpoint. Additional software such as CRIU may be required to // restore a task from a checkpoint -func WithTaskCheckpoint(desc v1.Descriptor) NewTaskOpts { +func WithTaskCheckpoint(im Image) NewTaskOpts { return func(ctx context.Context, c *Client, info *TaskInfo) error { + desc := im.Target() id := desc.Digest index, err := decodeIndex(ctx, c.ContentStore(), id) if err != nil { diff --git a/task.go b/task.go index 86a00cc48..8f29b2ce2 100644 --- a/task.go +++ b/task.go @@ -17,6 +17,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/plugin" @@ -88,6 +89,7 @@ func WithStdinCloser(r *IOCloseInfo) { // CheckpointTaskInfo allows specific checkpoint information to be set for the task type CheckpointTaskInfo struct { + Name string // ParentCheckpoint is the digest of a parent checkpoint ParentCheckpoint digest.Digest // Options hold runtime specific settings for checkpointing a task @@ -124,7 +126,7 @@ type Task interface { // OCI Index that can be push and pulled from a remote resource. // // Additional software like CRIU maybe required to checkpoint and restore tasks - Checkpoint(context.Context, ...CheckpointTaskOpts) (v1.Descriptor, error) + Checkpoint(context.Context, ...CheckpointTaskOpts) (Image, error) // Update modifies executing tasks with updated settings Update(context.Context, ...UpdateTaskOpts) error // LoadProcess loads a previously created exec'd process @@ -350,46 +352,73 @@ func (t *task) Resize(ctx context.Context, w, h uint32) error { return errdefs.FromGRPC(err) } -func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (d v1.Descriptor, err error) { +func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Image, error) { request := &tasks.CheckpointTaskRequest{ ContainerID: t.id, } var i CheckpointTaskInfo for _, o := range opts { if err := o(&i); err != nil { - return d, err + return nil, err } } + const checkpointDateFormat = "01-02-2006-15:04:05" + // set a default name + if i.Name == "" { + i.Name = fmt.Sprintf("io.containerd/checkpoint/%s:%s", t.id, time.Now().Format(checkpointDateFormat)) + } request.ParentCheckpoint = i.ParentCheckpoint if i.Options != nil { any, err := typeurl.MarshalAny(i.Options) if err != nil { - return d, err + return nil, err } request.Options = any } // make sure we pause it and resume after all other filesystem operations are completed if err := t.Pause(ctx); err != nil { - return d, err + return nil, err } defer t.Resume(ctx) cr, err := t.client.ContainerService().Get(ctx, t.id) if err != nil { - return d, err + return nil, err + } + index := v1.Index{ + Annotations: make(map[string]string), } - var index v1.Index if err := t.checkpointTask(ctx, &index, request); err != nil { - return d, err + return nil, err } - if err := t.checkpointImage(ctx, &index, cr.Image); err != nil { - return d, err + if cr.Image != "" { + if err := t.checkpointImage(ctx, &index, cr.Image); err != nil { + return nil, err + } + index.Annotations["image.name"] = cr.Image } - if err := t.checkpointRWSnapshot(ctx, &index, cr.Snapshotter, cr.SnapshotKey); err != nil { - return d, err + if cr.SnapshotKey != "" { + if err := t.checkpointRWSnapshot(ctx, &index, cr.Snapshotter, cr.SnapshotKey); err != nil { + return nil, err + } } - index.Annotations = make(map[string]string) - index.Annotations["image.name"] = cr.Image - return t.writeIndex(ctx, &index) + desc, err := t.writeIndex(ctx, &index) + if err != nil { + return nil, err + } + im := images.Image{ + Name: i.Name, + Target: desc, + Labels: map[string]string{ + "containerd.io/checkpoint": "true", + }, + } + if im, err = t.client.ImageService().Create(ctx, im); err != nil { + return nil, err + } + return &image{ + client: t.client, + i: im, + }, nil } // UpdateTaskInfo allows updated specific settings to be changed on a task diff --git a/task_opts.go b/task_opts.go index dc0d0e787..261ccba64 100644 --- a/task_opts.go +++ b/task_opts.go @@ -28,6 +28,14 @@ func WithExit(r *CheckpointTaskInfo) error { return nil } +// WithCheckpointName sets the image name for the checkpoint +func WithCheckpointName(name string) CheckpointTaskOpts { + return func(r *CheckpointTaskInfo) error { + r.Name = name + return nil + } +} + // ProcessDeleteOpts allows the caller to set options for the deletion of a task type ProcessDeleteOpts func(context.Context, Process) error From e833da1356bef5ca76af4ccbb12c928dcd513481 Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 17 Oct 2017 16:53:44 -0400 Subject: [PATCH 4/5] Clear root labels during checkpoint Signed-off-by: Michael Crosby --- task.go | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/task.go b/task.go index 8f29b2ce2..8329a0b7f 100644 --- a/task.go +++ b/task.go @@ -387,6 +387,15 @@ func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Imag index := v1.Index{ Annotations: make(map[string]string), } + // make sure we clear the gc root labels reguardless of success + var clearRoots []ocispec.Descriptor + defer func() { + for _, r := range append(index.Manifests, clearRoots...) { + if err := clearRootGCLabel(ctx, t.client, r); err != nil { + log.G(ctx).WithError(err).WithField("dgst", r.Digest).Warnf("failed to remove root marker") + } + } + }() if err := t.checkpointTask(ctx, &index, request); err != nil { return nil, err } @@ -405,6 +414,7 @@ func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Imag if err != nil { return nil, err } + clearRoots = append(clearRoots, desc) im := images.Image{ Name: i.Name, Target: desc, @@ -552,24 +562,13 @@ func (t *task) writeIndex(ctx context.Context, index *v1.Index) (d v1.Descriptor labels := map[string]string{ "containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339), } - for i, m := range index.Manifests { labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = m.Digest.String() - defer func(m ocispec.Descriptor) { - if err == nil { - info := content.Info{Digest: m.Digest} - if _, uerr := t.client.ContentStore().Update(ctx, info, "labels.containerd.io/gc.root"); uerr != nil { - log.G(ctx).WithError(uerr).WithField("dgst", m.Digest).Warnf("failed to remove root marker") - } - } - }(m) } - buf := bytes.NewBuffer(nil) if err := json.NewEncoder(buf).Encode(index); err != nil { return v1.Descriptor{}, err } - return writeContent(ctx, t.client.ContentStore(), v1.MediaTypeImageIndex, t.id, buf, content.WithLabels(labels)) } @@ -592,3 +591,9 @@ func writeContent(ctx context.Context, store content.Store, mediaType, ref strin Size: size, }, nil } + +func clearRootGCLabel(ctx context.Context, client *Client, desc ocispec.Descriptor) error { + info := content.Info{Digest: desc.Digest} + _, err := client.ContentStore().Update(ctx, info, "labels.containerd.io/gc.root") + return err +} From e4c6bf3b5ecc6a7afb4f45b10edea40841af3a6c Mon Sep 17 00:00:00 2001 From: Michael Crosby Date: Tue, 17 Oct 2017 17:10:03 -0400 Subject: [PATCH 5/5] Remove default filter from ctr Signed-off-by: Michael Crosby --- cmd/ctr/images.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cmd/ctr/images.go b/cmd/ctr/images.go index c355f0c7e..bf8880f16 100644 --- a/cmd/ctr/images.go +++ b/cmd/ctr/images.go @@ -56,17 +56,6 @@ var imagesListCommand = cli.Command{ imageStore := client.ImageService() cs := client.ContentStore() - - if len(filters) == 0 { - filters = append(filters, `labels.containerd."io/checkpoint"!=true`) - } else { - for _, f := range filters { - if !strings.Contains(f, `labels.containerd."io/checkpoint"`) { - f += `,labels.containerd."io/checkpoint"!=true` - _ = f // ignore error: ineffectual assignment to f - } - } - } imageList, err := imageStore.List(ctx, filters...) if err != nil { return errors.Wrap(err, "failed to list images")