cr: support checkpoint/restore without image

support checkpoint without committing a checkpoint dir into a
checkpoint image and restore without untar image into checkpoint
directory. support for both v1 and v2 runtime

Signed-off-by: Ace-Tang <aceapril@126.com>

cmd/ctr/commands/tasks/checkpoint.go

@@ -36,6 +36,14 @@ var checkpointCommand = cli.Command{
 			Name:  "exit",
 			Usage: "stop the container after the checkpoint",
 		},
+		cli.StringFlag{
+			Name:  "image-path",
+			Usage: "path to criu image files",
+		},
+		cli.StringFlag{
+			Name:  "work-path",
+			Usage: "path to criu work files and logs",
+		},
 	},
 	Action: func(context *cli.Context) error {
 		id := context.Args().First()
@@ -59,40 +67,55 @@ var checkpointCommand = cli.Command{
 		if err != nil {
 			return err
 		}
-		var opts []containerd.CheckpointTaskOpts
+		opts := []containerd.CheckpointTaskOpts{withCheckpointOpts(info.Runtime.Name, context)}
-		if context.Bool("exit") {
-			opts = append(opts, withExit(info.Runtime.Name))
-		}
 		checkpoint, err := task.Checkpoint(ctx, opts...)
 		if err != nil {
 			return err
 		}
+		if context.String("image-path") == "" {
 			fmt.Println(checkpoint.Name())
+		}
 		return nil
 	},
 }
 
-func withExit(rt string) containerd.CheckpointTaskOpts {
+// withCheckpointOpts only suitable for runc runtime now
+func withCheckpointOpts(rt string, context *cli.Context) containerd.CheckpointTaskOpts {
 	return func(r *containerd.CheckpointTaskInfo) error {
+		imagePath := context.String("image-path")
+		workPath := context.String("work-path")
+
 		switch rt {
 		case "io.containerd.runc.v1":
 			if r.Options == nil {
-				r.Options = &options.CheckpointOptions{
+				r.Options = &options.CheckpointOptions{}
-					Exit: true,
 			}
-			} else {
 			opts, _ := r.Options.(*options.CheckpointOptions)
+
+			if context.Bool("exit") {
 				opts.Exit = true
 			}
-		default:
+			if imagePath != "" {
+				opts.ImagePath = imagePath
+			}
+			if workPath != "" {
+				opts.WorkPath = workPath
+			}
+		case "io.containerd.runtime.v1.linux":
 			if r.Options == nil {
-				r.Options = &runctypes.CheckpointOptions{
+				r.Options = &runctypes.CheckpointOptions{}
-					Exit: true,
 			}
-			} else {
 			opts, _ := r.Options.(*runctypes.CheckpointOptions)
+
+			if context.Bool("exit") {
 				opts.Exit = true
 			}
+			if imagePath != "" {
+				opts.ImagePath = imagePath
+			}
+			if workPath != "" {
+				opts.WorkPath = workPath
+			}
 		}
 		return nil
 	}

runtime/v1/linux/proc/init.go

@@ -76,6 +76,7 @@ type Init struct {
 	IoGID        int
 	NoPivotRoot  bool
 	NoNewKeyring bool
+	CriuWorkPath string
 }
 
 // NewRunc returns a new runc instance for a process
@@ -132,7 +133,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error {
 		opts := &runc.RestoreOpts{
 			CheckpointOpts: runc.CheckpointOpts{
 				ImagePath:  r.Checkpoint,
-				WorkDir:    p.WorkDir,
+				WorkDir:    p.CriuWorkPath,
 				ParentPath: r.ParentCheckpoint,
 			},
 			PidFile:     pidFile,
@@ -422,8 +423,12 @@ func (p *Init) checkpoint(ctx context.Context, r *CheckpointConfig) error {
 	if !r.Exit {
 		actions = append(actions, runc.LeaveRunning)
 	}
-	work := filepath.Join(p.WorkDir, "criu-work")
+	// keep criu work directory if criu work dir is set
+	work := r.WorkDir
+	if work == "" {
+		work = filepath.Join(p.WorkDir, "criu-work")
 		defer os.RemoveAll(work)
+	}
 	if err := p.runtime.Checkpoint(ctx, p.id, &runc.CheckpointOpts{
 		WorkDir:                  work,
 		ImagePath:                r.Path,

runtime/v1/linux/proc/types.go

@@ -55,6 +55,7 @@ type ExecConfig struct {
 
 // CheckpointConfig holds task checkpoint configuration
 type CheckpointConfig struct {
+	WorkDir                  string
 	Path                     string
 	Exit                     bool
 	AllowOpenTCP             bool

runtime/v1/shim/service.go

@@ -448,6 +448,7 @@ func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskReque
 		AllowTerminal:            options.Terminal,
 		FileLocks:                options.FileLocks,
 		EmptyNamespaces:          options.EmptyNamespaces,
+		WorkDir:                  options.WorkPath,
 	}); err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
@@ -657,5 +658,11 @@ func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu st
 	p.IoGID = int(options.IoGid)
 	p.NoPivotRoot = options.NoPivotRoot
 	p.NoNewKeyring = options.NoNewKeyring
+	p.CriuWorkPath = options.CriuWorkPath
+	if p.CriuWorkPath == "" {
+		// if criu work path not set, use container WorkDir
+		p.CriuWorkPath = p.WorkDir
+	}
+
 	return p, nil
 }

runtime/v2/runc/service.go

@@ -562,6 +562,7 @@ func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskReque
 		AllowTerminal:            opts.Terminal,
 		FileLocks:                opts.FileLocks,
 		EmptyNamespaces:          opts.EmptyNamespaces,
+		WorkDir:                  opts.WorkPath,
 	}); err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
@@ -806,5 +807,11 @@ func newInit(ctx context.Context, path, workDir, namespace string, platform rpro
 	p.IoGID = int(options.IoGid)
 	p.NoPivotRoot = options.NoPivotRoot
 	p.NoNewKeyring = options.NoNewKeyring
+	p.CriuWorkPath = options.CriuWorkPath
+	if p.CriuWorkPath == "" {
+		// if criu work path not set, use container WorkDir
+		p.CriuWorkPath = p.WorkDir
+	}
+
 	return p, nil
 }

services/tasks/local.go

@@ -41,7 +41,9 @@ import (
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/runtime"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
 	"github.com/containerd/containerd/runtime/v2"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/containerd/services"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -123,11 +125,16 @@ type local struct {
 }
 
 func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.CallOption) (*api.CreateTaskResponse, error) {
-	var (
+	container, err := l.getContainer(ctx, r.ContainerID)
-		checkpointPath string
+	if err != nil {
-		err            error
+		return nil, errdefs.ToGRPC(err)
-	)
+	}
-	if r.Checkpoint != nil {
+	checkpointPath, err := getRestorePath(container.Runtime.Name, r.Options)
+	if err != nil {
+		return nil, err
+	}
+	// jump get checkpointPath from checkpoint image
+	if checkpointPath != "" && r.Checkpoint != nil {
 		checkpointPath, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctrd-checkpoint")
 		if err != nil {
 			return nil, err
@@ -149,10 +156,6 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
 			return nil, err
 		}
 	}
-	container, err := l.getContainer(ctx, r.ContainerID)
-	if err != nil {
-		return nil, errdefs.ToGRPC(err)
-	}
 	opts := runtime.CreateOpts{
 		Spec: container.Spec,
 		IO: runtime.IO{
@@ -478,14 +481,27 @@ func (l *local) Checkpoint(ctx context.Context, r *api.CheckpointTaskRequest, _
 	if err != nil {
 		return nil, err
 	}
-	image, err := ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctd-checkpoint")
+	image, err := getCheckpointPath(container.Runtime.Name, r.Options)
+	if err != nil {
+		return nil, err
+	}
+	checkpointImageExists := false
+	if image == "" {
+		checkpointImageExists = true
+		image, err = ioutil.TempDir(os.Getenv("XDG_RUNTIME_DIR"), "ctd-checkpoint")
 		if err != nil {
 			return nil, errdefs.ToGRPC(err)
 		}
 		defer os.RemoveAll(image)
+	}
 	if err := t.Checkpoint(ctx, image, r.Options); err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
+	// do not commit checkpoint image if checkpoint ImagePath is passed,
+	// return if checkpointImageExists is false
+	if !checkpointImageExists {
+		return &api.CheckpointTaskResponse{}, nil
+	}
 	// write checkpoint to the content store
 	tar := archive.Diff(ctx, "", image)
 	cp, err := l.writeContent(ctx, images.MediaTypeContainerd1Checkpoint, image, tar)
@@ -663,3 +679,71 @@ func (l *local) allRuntimes() (o []runtime.PlatformRuntime) {
 	o = append(o, l.v2Runtime)
 	return o
 }
+
+// getCheckpointPath only suitable for runc runtime now
+func getCheckpointPath(runtime string, option *ptypes.Any) (string, error) {
+	if option == nil {
+		return "", nil
+	}
+
+	var checkpointPath string
+	switch runtime {
+	case "io.containerd.runc.v1":
+		v, err := typeurl.UnmarshalAny(option)
+		if err != nil {
+			return "", err
+		}
+		opts, ok := v.(*options.CheckpointOptions)
+		if !ok {
+			return "", fmt.Errorf("invalid task checkpoint option for %s", runtime)
+		}
+		checkpointPath = opts.ImagePath
+
+	case "io.containerd.runtime.v1.linux":
+		v, err := typeurl.UnmarshalAny(option)
+		if err != nil {
+			return "", err
+		}
+		opts, ok := v.(*runctypes.CheckpointOptions)
+		if !ok {
+			return "", fmt.Errorf("invalid task checkpoint option for %s", runtime)
+		}
+		checkpointPath = opts.ImagePath
+	}
+
+	return checkpointPath, nil
+}
+
+// getRestorePath only suitable for runc runtime now
+func getRestorePath(runtime string, option *ptypes.Any) (string, error) {
+	if option == nil {
+		return "", nil
+	}
+
+	var restorePath string
+	switch runtime {
+	case "io.containerd.runc.v1":
+		v, err := typeurl.UnmarshalAny(option)
+		if err != nil {
+			return "", err
+		}
+		opts, ok := v.(*options.Options)
+		if !ok {
+			return "", fmt.Errorf("invalid task create option for %s", runtime)
+		}
+		restorePath = opts.CriuImagePath
+
+	case "io.containerd.runtime.v1.linux":
+		v, err := typeurl.UnmarshalAny(option)
+		if err != nil {
+			return "", err
+		}
+		opts, ok := v.(*runctypes.CreateOptions)
+		if !ok {
+			return "", fmt.Errorf("invalid task create option for %s", runtime)
+		}
+		restorePath = opts.CriuImagePath
+	}
+
+	return restorePath, nil
+}

task.go

@@ -37,6 +37,8 @@ import (
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/rootfs"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/typeurl"
 	google_protobuf "github.com/gogo/protobuf/types"
 	digest "github.com/opencontainers/go-digest"
@@ -433,6 +435,11 @@ func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Imag
 	if err := t.checkpointTask(ctx, &index, request); err != nil {
 		return nil, err
 	}
+	// if checkpoint image path passed, jump checkpoint image
+	if isCheckpointPathExist(cr.Runtime.Name, i.Options) {
+		return nil, nil
+	}
+
 	if cr.Image != "" {
 		if err := t.checkpointImage(ctx, &index, cr.Image); err != nil {
 			return nil, err
@@ -542,6 +549,7 @@ func (t *task) checkpointTask(ctx context.Context, index *v1.Index, request *tas
 	if err != nil {
 		return errdefs.FromGRPC(err)
 	}
+	// NOTE: response.Descriptors can be an empty slice if checkpoint image is jumped
 	// add the checkpoint descriptors to the index
 	for _, d := range response.Descriptors {
 		index.Manifests = append(index.Manifests, v1.Descriptor{
@@ -619,3 +627,24 @@ func writeContent(ctx context.Context, store content.Ingester, mediaType, ref st
 		Size:      size,
 	}, nil
 }
+
+// isCheckpointPathExist only suitable for runc runtime now
+func isCheckpointPathExist(runtime string, v interface{}) bool {
+	if v == nil {
+		return false
+	}
+
+	switch runtime {
+	case "io.containerd.runc.v1":
+		if opts, ok := v.(*options.CheckpointOptions); ok && opts.ImagePath != "" {
+			return true
+		}
+
+	case "io.containerd.runtime.v1.linux":
+		if opts, ok := v.(*runctypes.CheckpointOptions); ok && opts.ImagePath != "" {
+			return true
+		}
+	}
+
+	return false
+}