From 45c700a95584594e869251178a88d751a5e3295b Mon Sep 17 00:00:00 2001 From: Evan Hazlett Date: Mon, 10 Sep 2018 17:16:52 +0000 Subject: [PATCH] refactor checkpoint and restore to client Signed-off-by: Evan Hazlett --- client.go | 105 ++++++++++++++++++ cmd/ctr/commands/containers/containers.go | 95 ++++++++++++++++ container.go | 100 +++++++++++++++++ container_checkpoint_opts.go | 125 ++++++++++++++++++++++ container_restore_opts.go | 95 ++++++++++++++++ images/mediatypes.go | 11 +- 6 files changed, 526 insertions(+), 5 deletions(-) create mode 100644 container_checkpoint_opts.go create mode 100644 container_restore_opts.go diff --git a/client.go b/client.go index fd20c3dd0..d13509b77 100644 --- a/client.go +++ b/client.go @@ -17,7 +17,9 @@ package containerd import ( + "bytes" "context" + "encoding/json" "fmt" "net/http" "runtime" @@ -36,6 +38,7 @@ import ( snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1" "github.com/containerd/containerd/api/services/tasks/v1" versionservice "github.com/containerd/containerd/api/services/version/v1" + "github.com/containerd/containerd/cio" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" contentproxy "github.com/containerd/containerd/content/proxy" @@ -46,6 +49,7 @@ import ( "github.com/containerd/containerd/leases" leasesproxy "github.com/containerd/containerd/leases/proxy" "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" "github.com/containerd/containerd/pkg/dialer" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" @@ -520,6 +524,107 @@ func (c *Client) ListImages(ctx context.Context, filters ...string) ([]Image, er return images, nil } +// Restore restores a container from a checkpoint +func (c *Client) Restore(ctx context.Context, id, ref string, opts ...RestoreOpts) error { + checkpoint, err := c.GetImage(ctx, ref) + if err != nil { + if !errdefs.IsNotFound(err) { + return err + } + ck, err := c.Fetch(ctx, ref) + if err != nil { + return err + } + checkpoint = NewImage(c, ck) + } + + store := c.ContentStore() + index, err := decodeIndex(ctx, store, checkpoint.Target()) + if err != nil { + return err + } + + // get image from annotation + imageName, ok := index.Annotations["image.name"] + if !ok { + return ErrCheckpointIndexImageNameNotFound + } + + image, err := c.Pull(ctx, imageName, WithPullUnpack) + if err != nil { + return err + } + + ctx, done, err := c.WithLease(ctx) + if err != nil { + return err + } + defer done(ctx) + + // container options + copts := []NewContainerOpts{ + WithNewSpec(oci.WithImageConfig(image)), + WithNewSnapshot(id, image), + } + topts := []NewTaskOpts{} + for _, o := range opts { + co, to, err := o(ctx, c, checkpoint, index) + if err != nil { + return err + } + copts = append(copts, co...) + topts = append(topts, to...) + } + + ctr, err := c.NewContainer(ctx, id, copts...) + if err != nil { + return err + } + + // apply rw layer + info, err := ctr.Info(ctx) + if err != nil { + return err + } + + rw, err := GetIndexByMediaType(index, ocispec.MediaTypeImageLayerGzip) + if err != nil { + return err + } + + mounts, err := c.SnapshotService(info.Snapshotter).Mounts(ctx, info.SnapshotKey) + if err != nil { + return err + } + + if _, err := c.DiffService().Apply(ctx, *rw, mounts); err != nil { + return err + } + + task, err := ctr.NewTask(ctx, cio.NewCreator(cio.WithStdio), topts...) + if err != nil { + return err + } + + if err := task.Start(ctx); err != nil { + return err + } + + return nil +} + +func writeIndex(ctx context.Context, index *ocispec.Index, client *Client, ref string) (d ocispec.Descriptor, err error) { + labels := map[string]string{} + for i, m := range index.Manifests { + labels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i)] = m.Digest.String() + } + data, err := json.Marshal(index) + if err != nil { + return ocispec.Descriptor{}, err + } + return writeContent(ctx, client.ContentStore(), ocispec.MediaTypeImageIndex, ref, bytes.NewReader(data), content.WithLabels(labels)) +} + // Subscribe to events that match one or more of the provided filters. // // Callers should listen on both the envelope and errs channels. If the errs diff --git a/cmd/ctr/commands/containers/containers.go b/cmd/ctr/commands/containers/containers.go index 0b88c6254..0b1d4dbbd 100644 --- a/cmd/ctr/commands/containers/containers.go +++ b/cmd/ctr/commands/containers/containers.go @@ -45,6 +45,8 @@ var Command = cli.Command{ infoCommand, listCommand, setLabelsCommand, + checkpointCommand, + restoreCommand, }, } @@ -282,3 +284,96 @@ var infoCommand = cli.Command{ return nil }, } + +var checkpointCommand = cli.Command{ + Name: "checkpoint", + Usage: "checkpoint a container", + ArgsUsage: "CONTAINER REF [flags]", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "rw", + Usage: "include the rw layer in the checkpoint", + }, + cli.BoolFlag{ + Name: "image", + Usage: "include the image in the checkpoint", + }, + cli.BoolFlag{ + Name: "task", + Usage: "checkpoint container task", + }, + }, + Action: func(context *cli.Context) error { + id := context.Args().First() + if id == "" { + return errors.New("container id must be provided") + } + ref := context.Args()[1] + if ref == "" { + return errors.New("ref must be provided") + } + client, ctx, cancel, err := commands.NewClient(context) + if err != nil { + return err + } + defer cancel() + opts := []containerd.CheckpointOpts{} + if context.Bool("image") { + opts = append(opts, containerd.WithCheckpointImage) + } + if context.Bool("rw") { + opts = append(opts, containerd.WithCheckpointRW) + } + if context.Bool("task") { + opts = append(opts, containerd.WithCheckpointTask) + } + container, err := client.LoadContainer(ctx, id) + if err != nil { + return err + } + if _, err := container.Checkpoint(ctx, ref, opts...); err != nil { + return err + } + + return nil + }, +} + +var restoreCommand = cli.Command{ + Name: "restore", + Usage: "restore a container from checkpoint", + ArgsUsage: "CONTAINER REF [flags]", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "live", + Usage: "restore the runtime and memory data from the checkpoint", + }, + }, + Action: func(context *cli.Context) error { + id := context.Args().First() + if id == "" { + return errors.New("container id must be provided") + } + ref := context.Args()[1] + if ref == "" { + return errors.New("ref must be provided") + } + opts := []containerd.RestoreOpts{ + containerd.WithRestoreSpec, + containerd.WithRestoreRuntime, + } + if context.Bool("live") { + opts = append(opts, containerd.WithRestoreLive) + } + client, ctx, cancel, err := commands.NewClient(context) + if err != nil { + return err + } + defer cancel() + if err := client.Restore(ctx, id, ref, opts...); err != nil { + return err + } + + return nil + }, +} diff --git a/container.go b/container.go index 3c09b2dbc..4d24adfc3 100644 --- a/container.go +++ b/container.go @@ -17,10 +17,12 @@ package containerd import ( + "bytes" "context" "encoding/json" "os" "path/filepath" + "runtime" "strings" "github.com/containerd/containerd/api/services/tasks/v1" @@ -28,9 +30,13 @@ import ( "github.com/containerd/containerd/cio" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/runtime/v2/runc/options" "github.com/containerd/typeurl" prototypes "github.com/gogo/protobuf/types" + ver "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -64,6 +70,8 @@ type Container interface { Extensions(context.Context) (map[string]prototypes.Any, error) // Update a container Update(context.Context, ...UpdateContainerOpts) error + // Checkpoint creates a checkpoint image of the current container + Checkpoint(context.Context, string, ...CheckpointOpts) (Image, error) } func containerFromRecord(client *Client, c containers.Container) *container { @@ -272,6 +280,98 @@ func (c *container) Update(ctx context.Context, opts ...UpdateContainerOpts) err return nil } +func (c *container) Checkpoint(ctx context.Context, ref string, opts ...CheckpointOpts) (Image, error) { + index := &ocispec.Index{ + Versioned: ver.Versioned{ + SchemaVersion: 2, + }, + Annotations: make(map[string]string), + } + copts := &options.CheckpointOptions{ + Exit: false, + OpenTcp: false, + ExternalUnixSockets: false, + Terminal: false, + FileLocks: true, + EmptyNamespaces: nil, + } + img, err := c.Image(ctx) + if err != nil { + return nil, err + } + index.Annotations["image.name"] = img.Name() + + ctx, done, err := c.client.WithLease(ctx) + if err != nil { + return nil, err + } + defer done(ctx) + + // pause task to checkpoint + if err := func(ctx context.Context) error { + task, err := c.Task(ctx, nil) + if err != nil { + if errdefs.IsNotFound(err) { + return nil + } + return err + } + if err := task.Pause(ctx); err != nil { + return err + } + defer task.Resume(ctx) + + info, err := c.Info(ctx) + if err != nil { + return err + } + + // add runtime info to index + index.Annotations["runtime.name"] = info.Runtime.Name + if info.Runtime.Options != nil { + data, err := info.Runtime.Options.Marshal() + if err != nil { + return err + } + r := bytes.NewReader(data) + desc, err := writeContent(ctx, c.client.ContentStore(), images.MediaTypeContainerd1CheckpointRuntimeOptions, info.ID+"-runtime-options", r) + if err != nil { + return err + } + desc.Platform = &ocispec.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + index.Manifests = append(index.Manifests, desc) + } + + // process remaining opts + for _, o := range opts { + if err := o(ctx, c.client, &info, index, copts); err != nil { + return err + } + } + return nil + }(ctx); err != nil { + return nil, err + } + + desc, err := writeIndex(ctx, index, c.client, c.ID()+"index") + if err != nil { + return nil, err + } + i := images.Image{ + Name: ref, + Target: desc, + } + checkpoint, err := c.client.ImageService().Create(ctx, i) + if err != nil { + return nil, err + } + + return NewImage(c.client, checkpoint), nil +} + func (c *container) loadTask(ctx context.Context, ioAttach cio.Attach) (Task, error) { response, err := c.client.TaskService().Get(ctx, &tasks.GetRequest{ ContainerID: c.id, diff --git a/container_checkpoint_opts.go b/container_checkpoint_opts.go new file mode 100644 index 000000000..0af5f8ce2 --- /dev/null +++ b/container_checkpoint_opts.go @@ -0,0 +1,125 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + "context" + "fmt" + "runtime" + + tasks "github.com/containerd/containerd/api/services/tasks/v1" + "github.com/containerd/containerd/containers" + "github.com/containerd/containerd/diff" + "github.com/containerd/containerd/rootfs" + "github.com/containerd/containerd/runtime/v2/runc/options" + "github.com/containerd/typeurl" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +var ( + // ErrCheckpointRWUnsupported is returned if the container runtime does not support checkpoint + ErrCheckpointRWUnsupported = errors.New("rw checkpoint is only supported on v2 runtimes") + // ErrMediaTypeNotFound returns an error when a media type in the manifest is unknown + ErrMediaTypeNotFound = errors.New("media type not found") + // ErrCheckpointIndexImageNameNotFound is returned when the checkpoint image name is not present in the index + ErrCheckpointIndexImageNameNotFound = errors.New("image name not present in index") + // ErrCheckpointIndexRuntimeNameNotFound is returned when the checkpoint runtime name is not present in the index + ErrCheckpointIndexRuntimeNameNotFound = errors.New("runtime name not present in index") +) + +// CheckpointOpts are options to manage the checkpoint operation +type CheckpointOpts func(context.Context, *Client, *containers.Container, *imagespec.Index, *options.CheckpointOptions) error + +// WithCheckpointImage includes the container image in the checkpoint +func WithCheckpointImage(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error { + ir, err := client.ImageService().Get(ctx, c.Image) + if err != nil { + return err + } + index.Manifests = append(index.Manifests, ir.Target) + return nil +} + +// WithCheckpointTask includes the running task +func WithCheckpointTask(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error { + any, err := typeurl.MarshalAny(copts) + if err != nil { + return nil + } + task, err := client.TaskService().Checkpoint(ctx, &tasks.CheckpointTaskRequest{ + ContainerID: c.ID, + Options: any, + }) + if err != nil { + return err + } + for _, d := range task.Descriptors { + index.Manifests = append(index.Manifests, imagespec.Descriptor{ + MediaType: d.MediaType, + Size: d.Size_, + Digest: d.Digest, + Platform: &imagespec.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + }, + }) + } + return nil +} + +// WithCheckpointRW includes the rw in the checkpoint +func WithCheckpointRW(ctx context.Context, client *Client, c *containers.Container, index *imagespec.Index, copts *options.CheckpointOptions) error { + cnt, err := client.LoadContainer(ctx, c.ID) + if err != nil { + return err + } + info, err := cnt.Info(ctx) + if err != nil { + return err + } + + diffOpts := []diff.Opt{ + diff.WithReference(fmt.Sprintf("checkpoint-rw-%s", info.SnapshotKey)), + } + rw, err := rootfs.CreateDiff(ctx, + info.SnapshotKey, + client.SnapshotService(info.Snapshotter), + client.DiffService(), + diffOpts..., + ) + if err != nil { + return err + + } + rw.Platform = &imagespec.Platform{ + OS: runtime.GOOS, + Architecture: runtime.GOARCH, + } + index.Manifests = append(index.Manifests, rw) + return nil +} + +// GetIndexByMediaType returns the index in a manifest for the specified media type +func GetIndexByMediaType(index *imagespec.Index, mt string) (*imagespec.Descriptor, error) { + for _, d := range index.Manifests { + if d.MediaType == mt { + return &d, nil + } + } + return nil, ErrMediaTypeNotFound +} diff --git a/container_restore_opts.go b/container_restore_opts.go new file mode 100644 index 000000000..52c343ec6 --- /dev/null +++ b/container_restore_opts.go @@ -0,0 +1,95 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package containerd + +import ( + "context" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/oci" + "github.com/containerd/typeurl" + "github.com/gogo/protobuf/proto" + ptypes "github.com/gogo/protobuf/types" + imagespec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// RestoreOpts are options to manage the restore operation +type RestoreOpts func(context.Context, *Client, Image, *imagespec.Index) ([]NewContainerOpts, []NewTaskOpts, error) + +// WithRestoreLive restores the runtime and memory data for the container +func WithRestoreLive(ctx context.Context, client *Client, checkpoint Image, index *imagespec.Index) ([]NewContainerOpts, []NewTaskOpts, error) { + return nil, []NewTaskOpts{ + WithTaskCheckpoint(checkpoint), + }, nil +} + +// WithRestoreRuntime restores the runtime for the container +func WithRestoreRuntime(ctx context.Context, client *Client, checkpoint Image, index *imagespec.Index) ([]NewContainerOpts, []NewTaskOpts, error) { + runtimeName, ok := index.Annotations["runtime.name"] + if !ok { + return nil, nil, ErrCheckpointIndexRuntimeNameNotFound + } + // restore options if present + m, err := GetIndexByMediaType(index, images.MediaTypeContainerd1CheckpointRuntimeOptions) + if err != nil { + if err != ErrMediaTypeNotFound { + return nil, nil, err + } + } + var options *ptypes.Any + if m != nil { + store := client.ContentStore() + data, err := content.ReadBlob(ctx, store, *m) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to read checkpoint runtime") + } + if err := proto.Unmarshal(data, options); err != nil { + return nil, nil, err + } + } + return []NewContainerOpts{ + WithRuntime(runtimeName, options), + }, nil, nil +} + +// WithRestoreSpec restores the spec from the checkpoint for the container +func WithRestoreSpec(ctx context.Context, client *Client, checkpoint Image, index *imagespec.Index) ([]NewContainerOpts, []NewTaskOpts, error) { + m, err := GetIndexByMediaType(index, images.MediaTypeContainerd1CheckpointConfig) + if err != nil { + return nil, nil, err + } + store := client.ContentStore() + data, err := content.ReadBlob(ctx, store, *m) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to read checkpoint config") + } + var any ptypes.Any + if err := proto.Unmarshal(data, &any); err != nil { + return nil, nil, err + } + + v, err := typeurl.UnmarshalAny(&any) + if err != nil { + return nil, nil, err + } + spec := v.(*oci.Spec) + return []NewContainerOpts{ + WithSpec(spec), + }, nil, nil +} diff --git a/images/mediatypes.go b/images/mediatypes.go index ca4ca071b..e7a84526f 100644 --- a/images/mediatypes.go +++ b/images/mediatypes.go @@ -29,11 +29,12 @@ const ( MediaTypeDockerSchema2Manifest = "application/vnd.docker.distribution.manifest.v2+json" MediaTypeDockerSchema2ManifestList = "application/vnd.docker.distribution.manifest.list.v2+json" // Checkpoint/Restore Media Types - MediaTypeContainerd1Checkpoint = "application/vnd.containerd.container.criu.checkpoint.criu.tar" - MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar" - MediaTypeContainerd1Resource = "application/vnd.containerd.container.resource.tar" - MediaTypeContainerd1RW = "application/vnd.containerd.container.rw.tar" - MediaTypeContainerd1CheckpointConfig = "application/vnd.containerd.container.checkpoint.config.v1+proto" + MediaTypeContainerd1Checkpoint = "application/vnd.containerd.container.criu.checkpoint.criu.tar" + MediaTypeContainerd1CheckpointPreDump = "application/vnd.containerd.container.criu.checkpoint.predump.tar" + MediaTypeContainerd1Resource = "application/vnd.containerd.container.resource.tar" + MediaTypeContainerd1RW = "application/vnd.containerd.container.rw.tar" + MediaTypeContainerd1CheckpointConfig = "application/vnd.containerd.container.checkpoint.config.v1+proto" + MediaTypeContainerd1CheckpointRuntimeOptions = "application/vnd.containerd.container.checkpoint.runtime.options+proto" // Legacy Docker schema1 manifest MediaTypeDockerSchema1Manifest = "application/vnd.docker.distribution.manifest.v1+prettyjws" )