github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add checkpoint and restore

Browse Source 
Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update go-runc to 49b2a02ec1ed3e4ae52d30b54a291b75

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add shim to restore creation

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Keep checkpoint path in service

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Add C/R to non-shim build

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Checkpoint rw and image

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Pause container on bind checkpoints

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Return dump.log in error on checkpoint failure

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Pause container for checkpoint

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Update runc to 639454475cb9c8b861cc599f8bcd5c8c790ae402

For checkpoint into to work you need runc version
639454475cb9c8b861cc599f8bcd5c8c790ae402 + and criu 3.0 as this is what
I have been testing with.

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Move restore behind create calls

This remove the restore RPCs in favor of providing the checkpoint
information to the `Create` calls of a container.  If provided, the
container will be created/restored from the checkpoint instead of an
existing container.

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>

Regen protos after rebase

Signed-off-by: Michael Crosby <crosbymichael@gmail.com>
This commit is contained in:
Michael Crosby
2017-05-12 10:18:00 -07:00
parent 5ee77fc281
commit 7cc1b64bd8
31 changed files with 2153 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
cmd/ctr/checkpoint.go Normal file
View File

@@ -0,0 +1,196 @@
package main
import (
"bytes"
gocontext "context"
"encoding/json"
"fmt"
"io"
"runtime"
"github.com/Sirupsen/logrus"
"github.com/containerd/containerd/api/services/execution"
"github.com/containerd/containerd/api/types/descriptor"
"github.com/containerd/containerd/archive"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/rootfs"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
specs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/errors"
"github.com/urfave/cli"
)
var checkpointCommand = cli.Command{
Name: "checkpoint",
Usage: "checkpoint a container",
Flags: []cli.Flag{
cli.StringFlag{
Name: "id",
Usage: "id of the container",
},
cli.BoolFlag{
Name: "exit",
Usage: "stop the container after the checkpoint",
},
cli.BoolFlag{
Name: "binds",
Usage: "checkpoint bind mounts with the checkpoint",
},
},
Action: func(context *cli.Context) error {
var (
id = context.String("id")
ctx = gocontext.Background()
)
if id == "" {
return errors.New("container id must be provided")
}
containers, err := getExecutionService(context)
if err != nil {
return err
}
content, err := getContentStore(context)
if err != nil {
return err
}
imageStore, err := getImageStore(context)
if err != nil {
return errors.Wrap(err, "failed resolving image store")
}
var spec specs.Spec
info, err := containers.Info(ctx, &execution.InfoRequest{
ID: id,
})
if err != nil {
return err
}
if err := json.Unmarshal(info.Spec.Value, &spec); err != nil {
return err
}
stopped := context.Bool("exit")
// if the container will still be running after the checkpoint make sure that
// we pause the container and give us time to checkpoint the filesystem before
// it resumes execution
if !stopped {
if _, err := containers.Pause(ctx, &execution.PauseRequest{
ID: id,
}); err != nil {
return err
}
defer func() {
if _, err := containers.Resume(ctx, &execution.ResumeRequest{
ID: id,
}); err != nil {
logrus.WithError(err).Error("ctr: unable to resume container")
}
}()
}
checkpoint, err := containers.Checkpoint(ctx, &execution.CheckpointRequest{
ID: id,
Exit: context.Bool("exit"),
})
if err != nil {
return err
}
image, err := imageStore.Get(ctx, spec.Annotations["image"])
if err != nil {
return err
}
var additionalDescriptors []*descriptor.Descriptor
if context.Bool("binds") {
if additionalDescriptors, err = checkpointBinds(ctx, &spec, content); err != nil {
return err
}
}
var index ocispec.ImageIndex
for _, d := range append(checkpoint.Descriptors, additionalDescriptors...) {
index.Manifests = append(index.Manifests, ocispec.ManifestDescriptor{
Descriptor: ocispec.Descriptor{
MediaType: d.MediaType,
Size: d.Size_,
Digest: d.Digest,
},
Platform: ocispec.Platform{
OS: runtime.GOOS,
Architecture: runtime.GOARCH,
},
})
}
// add image to the index
index.Manifests = append(index.Manifests, ocispec.ManifestDescriptor{
Descriptor: image.Target,
})
// checkpoint rw layer
snapshotter, err := getSnapshotter(context)
if err != nil {
return err
}
differ, err := getDiffService(context)
if err != nil {
return err
}
rw, err := rootfs.Diff(ctx, id, fmt.Sprintf("checkpoint-rw-%s", id), snapshotter, differ)
if err != nil {
return err
}
index.Manifests = append(index.Manifests, ocispec.ManifestDescriptor{
Descriptor: rw,
Platform: ocispec.Platform{
OS: runtime.GOOS,
Architecture: runtime.GOARCH,
},
})
data, err := json.Marshal(index)
if err != nil {
return err
}
// write the index to the content store
buf := bytes.NewReader(data)
desc, err := writeContent(ctx, content, ocispec.MediaTypeImageIndex, id, buf)
if err != nil {
return err
}
fmt.Println(desc.Digest.String())
return nil
},
}
func checkpointBinds(ctx gocontext.Context, s *specs.Spec, store content.Store) ([]*descriptor.Descriptor, error) {
var out []*descriptor.Descriptor
for _, m := range s.Mounts {
if m.Type != "bind" {
continue
}
tar := archive.Diff(ctx, "", m.Source)
d, err := writeContent(ctx, store, images.MediaTypeContainerd1Resource, m.Source, tar)
if err := tar.Close(); err != nil {
return nil, err
}
if err != nil {
return nil, err
}
out = append(out, d)
}
return out, nil
}
func writeContent(ctx gocontext.Context, store content.Store, mediaType, ref string, r io.Reader) (*descriptor.Descriptor, error) {
writer, err := store.Writer(ctx, ref, 0, "")
if err != nil {
return nil, err
}
defer writer.Close()
size, err := io.Copy(writer, r)
if err != nil {
return nil, err
}
if err := writer.Commit(0, ""); err != nil {
return nil, err
}
return &descriptor.Descriptor{
MediaType: mediaType,
Digest: writer.Digest(),
Size_: size,
}, nil
}

1
cmd/ctr/main.go
View File

@@ -51,6 +51,7 @@ containerd client
},
}
app.Commands = []cli.Command{
checkpointCommand,
runCommand,
eventsCommand,
deleteCommand,

189
cmd/ctr/run.go
View File

@@ -4,6 +4,7 @@ import (
gocontext "context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"runtime"
@@ -11,9 +12,9 @@ import (
"github.com/containerd/console"
"github.com/containerd/containerd"
"github.com/containerd/containerd/api/services/execution"
mounttypes "github.com/containerd/containerd/api/types/mount"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/snapshot"
digest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
@@ -55,8 +56,12 @@ var runCommand = cli.Command{
Usage: "enable host networking for the container",
},
cli.BoolFlag{
Name: "keep",
Usage: "keep container after running",
Name: "rm",
Usage: "remove the container after running",
},
cli.StringFlag{
Name: "checkpoint",
Usage: "provide the checkpoint digest to restore the container",
},
},
Action: func(context *cli.Context) error {
@@ -94,82 +99,137 @@ var runCommand = cli.Command{
if err != nil {
return err
}
imageStore, err := getImageStore(context)
if err != nil {
return errors.Wrap(err, "failed resolving image store")
}
if runtime.GOOS != "windows" && context.String("rootfs") == "" {
ref := context.Args().First()
image, err := imageStore.Get(ctx, ref)
if err != nil {
return errors.Wrapf(err, "could not resolve %q", ref)
differ, err := getDiffService(context)
if err != nil {
return err
}
var (
checkpoint *ocispec.Descriptor
checkpointIndex digest.Digest
ref = context.Args().First()
)
if raw := context.String("checkpoint"); raw != "" {
if checkpointIndex, err = digest.Parse(raw); err != nil {
return err
}
// let's close out our db and tx so we don't hold the lock whilst running.
diffIDs, err := image.RootFS(ctx, content)
}
var spec []byte
if checkpointIndex != "" {
var index ocispec.ImageIndex
r, err := content.Reader(ctx, checkpointIndex)
if err != nil {
return err
}
if context.Bool("readonly") {
mounts, err = snapshotter.View(ctx, id, identity.ChainID(diffIDs).String())
} else {
mounts, err = snapshotter.Prepare(ctx, id, identity.ChainID(diffIDs).String())
}
err = json.NewDecoder(r).Decode(&index)
r.Close()
if err != nil {
if !snapshot.IsExist(err) {
return err
return err
}
var rw ocispec.Descriptor
for _, m := range index.Manifests {
switch m.MediaType {
case images.MediaTypeContainerd1Checkpoint:
fkingo := m.Descriptor
checkpoint = &fkingo
case images.MediaTypeContainerd1CheckpointConfig:
if r, err = content.Reader(ctx, m.Digest); err != nil {
return err
}
spec, err = ioutil.ReadAll(r)
r.Close()
if err != nil {
return err
}
case images.MediaTypeDockerSchema2Manifest:
// make sure we have the original image that was used during checkpoint
diffIDs, err := images.RootFS(ctx, content, m.Descriptor)
if err != nil {
return err
}
if _, err := snapshotter.Prepare(ctx, id, identity.ChainID(diffIDs).String()); err != nil {
if !snapshot.IsExist(err) {
return err
}
}
case ocispec.MediaTypeImageLayer:
rw = m.Descriptor
}
mounts, err = snapshotter.Mounts(ctx, id)
}
if mounts, err = snapshotter.Mounts(ctx, id); err != nil {
return err
}
if _, err := differ.Apply(ctx, rw, mounts); err != nil {
return err
}
} else {
if runtime.GOOS != "windows" && context.String("rootfs") == "" {
image, err := imageStore.Get(ctx, ref)
if err != nil {
return errors.Wrapf(err, "could not resolve %q", ref)
}
// let's close out our db and tx so we don't hold the lock whilst running.
diffIDs, err := image.RootFS(ctx, content)
if err != nil {
return err
}
} else {
if context.Bool("readonly") {
mounts, err = snapshotter.View(ctx, id, identity.ChainID(diffIDs).String())
} else {
mounts, err = snapshotter.Prepare(ctx, id, identity.ChainID(diffIDs).String())
}
defer func() {
if id != "" {
if err != nil || context.Bool("rm") {
if err := snapshotter.Remove(ctx, id); err != nil {
logrus.WithError(err).Errorf("failed to remove snapshot %q", id)
}
}
}()
}
ic, err := image.Config(ctx, content)
if err != nil {
return err
}
switch ic.MediaType {
case ocispec.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config:
r, err := content.Reader(ctx, ic.Digest)
if err != nil {
if !snapshot.IsExist(err) {
return err
}
mounts, err = snapshotter.Mounts(ctx, id)
if err != nil {
return err
}
}
ic, err := image.Config(ctx, content)
if err != nil {
return err
}
if err := json.NewDecoder(r).Decode(&imageConfig); err != nil {
switch ic.MediaType {
case ocispec.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config:
r, err := content.Reader(ctx, ic.Digest)
if err != nil {
return err
}
if err := json.NewDecoder(r).Decode(&imageConfig); err != nil {
r.Close()
return err
}
r.Close()
return err
default:
return fmt.Errorf("unknown image config media type %s", ic.MediaType)
}
r.Close()
default:
return fmt.Errorf("unknown image config media type %s", ic.MediaType)
} else {
// TODO: get the image / rootfs through the API once windows has a snapshotter
}
} else {
// TODO: get the image / rootfs through the API once windows has a snapshotter
}
create, err := newCreateRequest(context, &imageConfig.Config, id, tmpDir, context.String("rootfs"))
if err != nil {
return err
}
for _, m := range mounts {
create.Rootfs = append(create.Rootfs, &mounttypes.Mount{
Type: m.Type,
Source: m.Source,
Options: m.Options,
})
if len(spec) == 0 {
if spec, err = newSpec(context, &imageConfig.Config, ref); err != nil {
return err
}
}
create, err := newCreateRequest(context, id, tmpDir, checkpoint, mounts, spec)
if err != nil {
return err
}
var con console.Console
if create.Terminal {
@@ -179,7 +239,6 @@ var runCommand = cli.Command{
return err
}
}
fwg, err := prepareStdio(create.Stdin, create.Stdout, create.Stderr, create.Terminal)
if err != nil {
return err
@@ -188,35 +247,33 @@ var runCommand = cli.Command{
if err != nil {
return err
}
pid := response.Pid
if create.Terminal {
if err := handleConsoleResize(ctx, containers, response.ID, response.Pid, con); err != nil {
if err := handleConsoleResize(ctx, containers, id, pid, con); err != nil {
logrus.WithError(err).Error("console resize")
}
} else {
sigc := forwardAllSignals(containers, id)
defer stopCatch(sigc)
}
if _, err := containers.Start(ctx, &execution.StartRequest{
ID: response.ID,
}); err != nil {
return err
if checkpoint == nil {
if _, err := containers.Start(ctx, &execution.StartRequest{
ID: id,
}); err != nil {
return err
}
}
// Ensure we read all io only if container started successfully.
defer fwg.Wait()
status, err := waitContainer(events, response.ID, response.Pid)
status, err := waitContainer(events, id, pid)
if err != nil {
return err
}
if !context.Bool("keep") {
if _, err := containers.Delete(ctx, &execution.DeleteRequest{
ID: response.ID,
}); err != nil {
return err
}
} else {
// Don't remove snapshot
id = ""
if _, err := containers.Delete(ctx, &execution.DeleteRequest{
ID: response.ID,
}); err != nil {
return err
}
if status != 0 {
return cli.NewExitError("", int(status))

34
cmd/ctr/run_unix.go
View File

@@ -18,7 +18,10 @@ import (
"github.com/Sirupsen/logrus"
"github.com/containerd/console"
"github.com/containerd/containerd"
"github.com/containerd/containerd/api/services/execution"
"github.com/containerd/containerd/api/types/descriptor"
"github.com/containerd/containerd/api/types/mount"
protobuf "github.com/gogo/protobuf/types"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -254,20 +257,24 @@ func getConfig(context *cli.Context, imageConfig *ocispec.ImageConfig, rootfs st
return customSpec(config, rootfs)
}
func newCreateRequest(context *cli.Context, imageConfig *ocispec.ImageConfig, id, tmpDir string, rootfs string) (*execution.CreateRequest, error) {
s, err := getConfig(context, imageConfig, rootfs)
func newSpec(context *cli.Context, config *ocispec.ImageConfig, imageRef string) ([]byte, error) {
s, err := getConfig(context, config, context.String("rootfs"))
if err != nil {
return nil, err
}
data, err := json.Marshal(s)
if err != nil {
return nil, err
if s.Annotations == nil {
s.Annotations = make(map[string]string)
}
s.Annotations["image"] = imageRef
return json.Marshal(s)
}
func newCreateRequest(context *cli.Context, id, tmpDir string, checkpoint *ocispec.Descriptor, mounts []containerd.Mount, spec []byte) (*execution.CreateRequest, error) {
create := &execution.CreateRequest{
ID: id,
Spec: &protobuf.Any{
TypeUrl: specs.Version,
Value: data,
Value: spec,
},
Runtime: context.String("runtime"),
Terminal: context.Bool("tty"),
@@ -275,7 +282,20 @@ func newCreateRequest(context *cli.Context, imageConfig *ocispec.ImageConfig, id
Stdout: filepath.Join(tmpDir, "stdout"),
Stderr: filepath.Join(tmpDir, "stderr"),
}
if checkpoint != nil {
create.Checkpoint = &descriptor.Descriptor{
MediaType: checkpoint.MediaType,
Size_: checkpoint.Size,
Digest: checkpoint.Digest,
}
}
for _, m := range mounts {
create.Rootfs = append(create.Rootfs, &mount.Mount{
Type: m.Type,
Source: m.Source,
Options: m.Options,
})
}
return create, nil
}

20
cmd/ctr/run_windows.go
View File

@@ -10,6 +10,7 @@ import (
"github.com/Sirupsen/logrus"
"github.com/containerd/console"
"github.com/containerd/containerd"
"github.com/containerd/containerd/api/services/execution"
"github.com/containerd/containerd/log"
"github.com/containerd/containerd/windows"
@@ -116,12 +117,15 @@ func getConfig(context *cli.Context, imageConfig *ocispec.ImageConfig, rootfs st
return s, nil
}
func newCreateRequest(context *cli.Context, imageConfig *ocispec.ImageConfig, id, tmpDir, rootfs string) (*execution.CreateRequest, error) {
spec, err := getConfig(context, imageConfig, rootfs)
func newSpec(context *cli.Context, config *ocispec.ImageConfig, imageRef string) ([]byte, error) {
spec, err := getConfig(context, config, context.String("rootfs"))
if err != nil {
return nil, err
}
if spec.Annotations == nil {
spec.Annotations = make(map[string]string)
}
spec.Annotations["image"] = imageRef
rtSpec := windows.RuntimeSpec{
OCISpec: *spec,
Configuration: hcs.Configuration{
@@ -129,16 +133,15 @@ func newCreateRequest(context *cli.Context, imageConfig *ocispec.ImageConfig, id
IgnoreFlushesDuringBoot: true,
AllowUnqualifiedDNSQuery: true},
}
return json.Marshal(rtSpec)
}
data, err := json.Marshal(rtSpec)
if err != nil {
return nil, err
}
func newCreateRequest(context *cli.Context, id, tmpDir string, checkpoint *ocispec.Descriptor, mounts []containerd.Mount, spec []byte) (*execution.CreateRequest, error) {
create := &execution.CreateRequest{
ID: id,
Spec: &protobuf.Any{
TypeUrl: specs.Version,
Value: data,
Value: spec,
},
Runtime: context.String("runtime"),
Terminal: context.Bool("tty"),
@@ -148,7 +151,6 @@ func newCreateRequest(context *cli.Context, imageConfig *ocispec.ImageConfig, id
if !create.Terminal {
create.Stderr = fmt.Sprintf(`%s\ctr-%s-stderr`, pipeRoot, id)
}
return create, nil
}