diff --git a/cmd/ctr/commands/tasks/checkpoint.go b/cmd/ctr/commands/tasks/checkpoint.go index 4d580ef12..94309e38c 100644 --- a/cmd/ctr/commands/tasks/checkpoint.go +++ b/cmd/ctr/commands/tasks/checkpoint.go @@ -36,6 +36,14 @@ var checkpointCommand = cli.Command{ Name: "exit", Usage: "stop the container after the checkpoint", }, + cli.StringFlag{ + Name: "image-path", + Usage: "path to criu image files", + }, + cli.StringFlag{ + Name: "work-path", + Usage: "path to criu work files and logs", + }, }, Action: func(context *cli.Context) error { id := context.Args().First() @@ -59,40 +67,55 @@ var checkpointCommand = cli.Command{ if err != nil { return err } - var opts []containerd.CheckpointTaskOpts - if context.Bool("exit") { - opts = append(opts, withExit(info.Runtime.Name)) - } + opts := []containerd.CheckpointTaskOpts{withCheckpointOpts(info.Runtime.Name, context)} checkpoint, err := task.Checkpoint(ctx, opts...) if err != nil { return err } - fmt.Println(checkpoint.Name()) + if context.String("image-path") == "" { + fmt.Println(checkpoint.Name()) + } return nil }, } -func withExit(rt string) containerd.CheckpointTaskOpts { +// withCheckpointOpts only suitable for runc runtime now +func withCheckpointOpts(rt string, context *cli.Context) containerd.CheckpointTaskOpts { return func(r *containerd.CheckpointTaskInfo) error { + imagePath := context.String("image-path") + workPath := context.String("work-path") + switch rt { case "io.containerd.runc.v1": if r.Options == nil { - r.Options = &options.CheckpointOptions{ - Exit: true, - } - } else { - opts, _ := r.Options.(*options.CheckpointOptions) + r.Options = &options.CheckpointOptions{} + } + opts, _ := r.Options.(*options.CheckpointOptions) + + if context.Bool("exit") { opts.Exit = true } - default: + if imagePath != "" { + opts.ImagePath = imagePath + } + if workPath != "" { + opts.WorkPath = workPath + } + case "io.containerd.runtime.v1.linux": if r.Options == nil { - r.Options = &runctypes.CheckpointOptions{ - Exit: true, - } - } else { - opts, _ := r.Options.(*runctypes.CheckpointOptions) + r.Options = &runctypes.CheckpointOptions{} + } + opts, _ := r.Options.(*runctypes.CheckpointOptions) + + if context.Bool("exit") { opts.Exit = true } + if imagePath != "" { + opts.ImagePath = imagePath + } + if workPath != "" { + opts.WorkPath = workPath + } } return nil } diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go index fa23b5e88..fe804ed81 100644 --- a/runtime/v1/linux/proc/init.go +++ b/runtime/v1/linux/proc/init.go @@ -76,6 +76,7 @@ type Init struct { IoGID int NoPivotRoot bool NoNewKeyring bool + CriuWorkPath string } // NewRunc returns a new runc instance for a process @@ -132,7 +133,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { opts := &runc.RestoreOpts{ CheckpointOpts: runc.CheckpointOpts{ ImagePath: r.Checkpoint, - WorkDir: p.WorkDir, + WorkDir: p.CriuWorkPath, ParentPath: r.ParentCheckpoint, }, PidFile: pidFile, @@ -422,8 +423,12 @@ func (p *Init) checkpoint(ctx context.Context, r *CheckpointConfig) error { if !r.Exit { actions = append(actions, runc.LeaveRunning) } - work := filepath.Join(p.WorkDir, "criu-work") - defer os.RemoveAll(work) + // keep criu work directory if criu work dir is set + work := r.WorkDir + if work == "" { + work = filepath.Join(p.WorkDir, "criu-work") + defer os.RemoveAll(work) + } if err := p.runtime.Checkpoint(ctx, p.id, &runc.CheckpointOpts{ WorkDir: work, ImagePath: r.Path, diff --git a/runtime/v1/linux/proc/types.go b/runtime/v1/linux/proc/types.go index 2bea98dc8..5d705c030 100644 --- a/runtime/v1/linux/proc/types.go +++ b/runtime/v1/linux/proc/types.go @@ -55,6 +55,7 @@ type ExecConfig struct { // CheckpointConfig holds task checkpoint configuration type CheckpointConfig struct { + WorkDir string Path string Exit bool AllowOpenTCP bool diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go index df6d8b64e..701dab435 100644 --- a/runtime/v1/shim/service.go +++ b/runtime/v1/shim/service.go @@ -448,6 +448,7 @@ func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskReque AllowTerminal: options.Terminal, FileLocks: options.FileLocks, EmptyNamespaces: options.EmptyNamespaces, + WorkDir: options.WorkPath, }); err != nil { return nil, errdefs.ToGRPC(err) } @@ -657,5 +658,11 @@ func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu st p.IoGID = int(options.IoGid) p.NoPivotRoot = options.NoPivotRoot p.NoNewKeyring = options.NoNewKeyring + p.CriuWorkPath = options.CriuWorkPath + if p.CriuWorkPath == "" { + // if criu work path not set, use container WorkDir + p.CriuWorkPath = p.WorkDir + } + return p, nil } diff --git a/runtime/v2/runc/service.go b/runtime/v2/runc/service.go index cb6f5e552..3e074d960 100644 --- a/runtime/v2/runc/service.go +++ b/runtime/v2/runc/service.go @@ -562,6 +562,7 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque AllowTerminal: opts.Terminal, FileLocks: opts.FileLocks, EmptyNamespaces: opts.EmptyNamespaces, + WorkDir: opts.WorkPath, }); err != nil { return nil, errdefs.ToGRPC(err) } @@ -806,5 +807,11 @@ func newInit(ctx context.Context, path, workDir, namespace string, platform rpro p.IoGID = int(options.IoGid) p.NoPivotRoot = options.NoPivotRoot p.NoNewKeyring = options.NoNewKeyring + p.CriuWorkPath = options.CriuWorkPath + if p.CriuWorkPath == "" { + // if criu work path not set, use container WorkDir + p.CriuWorkPath = p.WorkDir + } + return p, nil } diff --git a/services/tasks/local.go b/services/tasks/local.go index 773feeb4f..ee7dc1bc9 100644 --- a/services/tasks/local.go +++ b/services/tasks/local.go @@ -41,7 +41,9 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/runtime" + "github.com/containerd/containerd/runtime/linux/runctypes" "github.com/containerd/containerd/runtime/v2" + "github.com/containerd/containerd/runtime/v2/runc/options" "github.com/containerd/containerd/services" "github.com/containerd/typeurl" ptypes "github.com/gogo/protobuf/types" @@ -123,11 +125,16 @@ type local struct { } func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.CallOption) (*api.CreateTaskResponse, error) { - var ( - checkpointPath string - err error - ) - if r.Checkpoint != nil { + container, err := l.getContainer(ctx, r.ContainerID) + if err != nil { + return nil, errdefs.ToGRPC(err) + } + checkpointPath, err := getRestorePath(container.Runtime.Name, r.Options) + if err != nil { + return nil, err + } + // jump get checkpointPath from checkpoint image + if checkpointPath != "" && r.Checkpoint != nil { checkpointPath, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctrd-checkpoint") if err != nil { return nil, err @@ -149,10 +156,6 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc. return nil, err } } - container, err := l.getContainer(ctx, r.ContainerID) - if err != nil { - return nil, errdefs.ToGRPC(err) - } opts := runtime.CreateOpts{ Spec: container.Spec, IO: runtime.IO{ @@ -478,14 +481,27 @@ func (l *local) Checkpoint(ctx context.Context, r *api.CheckpointTaskRequest, _ if err != nil { return nil, err } - image, err := ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctd-checkpoint") + image, err := getCheckpointPath(container.Runtime.Name, r.Options) if err != nil { - return nil, errdefs.ToGRPC(err) + return nil, err + } + checkpointImageExists := false + if image == "" { + checkpointImageExists = true + image, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctd-checkpoint") + if err != nil { + return nil, errdefs.ToGRPC(err) + } + defer os.RemoveAll(image) } - defer os.RemoveAll(image) if err := t.Checkpoint(ctx, image, r.Options); err != nil { return nil, errdefs.ToGRPC(err) } + // do not commit checkpoint image if checkpoint ImagePath is passed, + // return if checkpointImageExists is false + if !checkpointImageExists { + return &api.CheckpointTaskResponse{}, nil + } // write checkpoint to the content store tar := archive.Diff(ctx, "", image) cp, err := l.writeContent(ctx, images.MediaTypeContainerd1Checkpoint, image, tar) @@ -663,3 +679,71 @@ func (l *local) allRuntimes() (o []runtime.PlatformRuntime) { o = append(o, l.v2Runtime) return o } + +// getCheckpointPath only suitable for runc runtime now +func getCheckpointPath(runtime string, option *ptypes.Any) (string, error) { + if option == nil { + return "", nil + } + + var checkpointPath string + switch runtime { + case "io.containerd.runc.v1": + v, err := typeurl.UnmarshalAny(option) + if err != nil { + return "", err + } + opts, ok := v.(*options.CheckpointOptions) + if !ok { + return "", fmt.Errorf("invalid task checkpoint option for %s", runtime) + } + checkpointPath = opts.ImagePath + + case "io.containerd.runtime.v1.linux": + v, err := typeurl.UnmarshalAny(option) + if err != nil { + return "", err + } + opts, ok := v.(*runctypes.CheckpointOptions) + if !ok { + return "", fmt.Errorf("invalid task checkpoint option for %s", runtime) + } + checkpointPath = opts.ImagePath + } + + return checkpointPath, nil +} + +// getRestorePath only suitable for runc runtime now +func getRestorePath(runtime string, option *ptypes.Any) (string, error) { + if option == nil { + return "", nil + } + + var restorePath string + switch runtime { + case "io.containerd.runc.v1": + v, err := typeurl.UnmarshalAny(option) + if err != nil { + return "", err + } + opts, ok := v.(*options.Options) + if !ok { + return "", fmt.Errorf("invalid task create option for %s", runtime) + } + restorePath = opts.CriuImagePath + + case "io.containerd.runtime.v1.linux": + v, err := typeurl.UnmarshalAny(option) + if err != nil { + return "", err + } + opts, ok := v.(*runctypes.CreateOptions) + if !ok { + return "", fmt.Errorf("invalid task create option for %s", runtime) + } + restorePath = opts.CriuImagePath + } + + return restorePath, nil +} diff --git a/task.go b/task.go index 6806e1162..e80c063e5 100644 --- a/task.go +++ b/task.go @@ -37,6 +37,8 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/plugin" "github.com/containerd/containerd/rootfs" + "github.com/containerd/containerd/runtime/linux/runctypes" + "github.com/containerd/containerd/runtime/v2/runc/options" "github.com/containerd/typeurl" google_protobuf "github.com/gogo/protobuf/types" digest "github.com/opencontainers/go-digest" @@ -433,6 +435,11 @@ func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Imag if err := t.checkpointTask(ctx, &index, request); err != nil { return nil, err } + // if checkpoint image path passed, jump checkpoint image + if isCheckpointPathExist(cr.Runtime.Name, i.Options) { + return nil, nil + } + if cr.Image != "" { if err := t.checkpointImage(ctx, &index, cr.Image); err != nil { return nil, err @@ -542,6 +549,7 @@ func (t *task) checkpointTask(ctx context.Context, index *v1.Index, request *tas if err != nil { return errdefs.FromGRPC(err) } + // NOTE: response.Descriptors can be an empty slice if checkpoint image is jumped // add the checkpoint descriptors to the index for _, d := range response.Descriptors { index.Manifests = append(index.Manifests, v1.Descriptor{ @@ -619,3 +627,24 @@ func writeContent(ctx context.Context, store content.Ingester, mediaType, ref st Size: size, }, nil } + +// isCheckpointPathExist only suitable for runc runtime now +func isCheckpointPathExist(runtime string, v interface{}) bool { + if v == nil { + return false + } + + switch runtime { + case "io.containerd.runc.v1": + if opts, ok := v.(*options.CheckpointOptions); ok && opts.ImagePath != "" { + return true + } + + case "io.containerd.runtime.v1.linux": + if opts, ok := v.(*runctypes.CheckpointOptions); ok && opts.ImagePath != "" { + return true + } + } + + return false +}