Merge pull request #2434 from crosbymichael/shimv2

Runtime v2 (shim API)

.travis.yml

@@ -27,7 +27,8 @@ addons:
       - socat
 
 env:
-  - TRAVIS_GOOS=linux TRAVIS_CGO_ENABLED=1
+  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runc.v1 TRAVIS_CGO_ENABLED=1
+  - TRAVIS_GOOS=linux TEST_RUNTIME=io.containerd.runtime.v1.linux TRAVIS_CGO_ENABLED=1
   - TRAVIS_GOOS=darwin TRAVIS_CGO_ENABLED=0
 
 before_install:

Makefile

@@ -172,6 +172,10 @@ bin/containerd-shim: cmd/containerd-shim FORCE # set !cgo and omit pie for a sta
 	@echo "$(WHALE) bin/containerd-shim"
 	@CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/containerd-shim ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim
 
+bin/containerd-shim-runc-v1: cmd/containerd-shim-runc-v1 FORCE # set !cgo and omit pie for a static shim build: https://github.com/golang/go/issues/17789#issuecomment-258542220
+	@echo "$(WHALE) bin/containerd-shim-runc-v1"
+	@CGO_ENABLED=0 go build ${GO_BUILD_FLAGS} -o bin/containerd-shim-runc-v1 ${SHIM_GO_LDFLAGS} ${GO_TAGS} ./cmd/containerd-shim-runc-v1
+
 binaries: $(BINARIES) ## build binaries
 	@echo "$(WHALE) $@"
 

Makefile.linux

@@ -14,7 +14,7 @@
 
 
 #linux specific settings
-COMMANDS += containerd-shim
+COMMANDS += containerd-shim containerd-shim-runc-v1
 
 # check GOOS for cross compile builds
 ifeq ($(GOOS),linux)

Protobuild.toml

@@ -37,7 +37,7 @@ plugins = ["fieldpath"] # disable grpc for this package
 
 [[overrides]]
 # enable ttrpc and disable fieldpath and grpc for the shim
-prefixes = ["github.com/containerd/containerd/runtime/shim/v1"]
+prefixes = ["github.com/containerd/containerd/runtime/v1/shim/v1", "github.com/containerd/containerd/runtime/v2/task"]
 plugins = ["ttrpc"]
 
 # Aggregrate the API descriptors to lock down API changes.

client.go

@@ -79,8 +79,12 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 			return nil, err
 		}
 	}
+	rt := fmt.Sprintf("%s.%s", plugin.RuntimePlugin, runtime.GOOS)
+	if copts.defaultRuntime != "" {
+		rt = copts.defaultRuntime
+	}
 	c := &Client{
-		runtime: fmt.Sprintf("%s.%s", plugin.RuntimePlugin, runtime.GOOS),
+		runtime: rt,
 	}
 	if copts.services != nil {
 		c.services = *copts.services

client_opts.go

@@ -24,6 +24,7 @@ import (
 
 type clientOpts struct {
 	defaultns      string
+	defaultRuntime string
 	services       *services
 	dialOptions    []grpc.DialOption
 }
@@ -42,6 +43,14 @@ func WithDefaultNamespace(ns string) ClientOpt {
 	}
 }
 
+// WithDefaultRuntime sets the default runtime on the client
+func WithDefaultRuntime(rt string) ClientOpt {
+	return func(c *clientOpts) error {
+		c.defaultRuntime = rt
+		return nil
+	}
+}
+
 // WithDialOpts allows grpc.DialOptions to be set on the connection
 func WithDialOpts(opts []grpc.DialOption) ClientOpt {
 	return func(c *clientOpts) error {

client_test.go

@@ -153,6 +153,9 @@ func newClient(t testing.TB, address string, opts ...ClientOpt) (*Client, error)
 	if testing.Short() {
 		t.Skip()
 	}
+	if rt := os.Getenv("TEST_RUNTIME"); rt != "" {
+		opts = append(opts, WithDefaultRuntime(rt))
+	}
 	// testutil.RequiresRoot(t) is not needed here (already called in TestMain)
 	return New(address, opts...)
 }

cmd/containerd-shim-runc-v1/main.go

@@ -0,0 +1,36 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/containerd/containerd/runtime/v2/runc"
+	"github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	if err := shim.Run(runc.New); err != nil {
+		logrus.WithError(err).Error("shim run")
+		fmt.Fprintf(os.Stderr, "containerd-shim-run-v1: %s\n", err)
+		os.Exit(1)
+	}
+}

cmd/containerd-shim/main_unix.go

@@ -36,9 +36,9 @@ import (
 
 	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/namespaces"
-	"github.com/containerd/containerd/runtime/linux/proc"
-	"github.com/containerd/containerd/runtime/shim"
-	shimapi "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v1/linux/proc"
+	"github.com/containerd/containerd/runtime/v1/shim"
+	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
 	"github.com/containerd/ttrpc"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -134,7 +134,7 @@ func executeShim() error {
 	shimapi.RegisterShimService(server, sv)
 
 	socket := socketFlag
-	if err := serve(server, socket); err != nil {
+	if err := serve(context.Background(), server, socket); err != nil {
 		return err
 	}
 	logger := logrus.WithFields(logrus.Fields{
@@ -152,7 +152,7 @@ func executeShim() error {
 
 // serve serves the ttrpc API over a unix socket at the provided path
 // this function does not block
-func serve(server *ttrpc.Server, path string) error {
+func serve(ctx context.Context, server *ttrpc.Server, path string) error {
 	var (
 		l   net.Listener
 		err error
@@ -172,7 +172,7 @@ func serve(server *ttrpc.Server, path string) error {
 	logrus.WithField("socket", path).Debug("serving api on unix socket")
 	go func() {
 		defer l.Close()
-		if err := server.Serve(context.Background(), l); err != nil &&
+		if err := server.Serve(ctx, l); err != nil &&
 			!strings.Contains(err.Error(), "use of closed network connection") {
 			logrus.WithError(err).Fatal("containerd-shim: ttrpc server failure")
 		}

cmd/containerd-shim/shim_darwin.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/containerd/containerd/runtime/shim"
+	"github.com/containerd/containerd/runtime/v1/shim"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/ttrpc"
 )

cmd/containerd-shim/shim_linux.go

@@ -20,7 +20,7 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/containerd/containerd/runtime/shim"
+	"github.com/containerd/containerd/runtime/v1/shim"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/ttrpc"
 	"github.com/opencontainers/runc/libcontainer/system"

cmd/containerd-shim/shim_unix.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"os/signal"
 
-	"github.com/containerd/containerd/runtime/shim"
+	"github.com/containerd/containerd/runtime/v1/shim"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/ttrpc"
 )

cmd/containerd/builtins_linux.go

@@ -19,7 +19,8 @@ package main
 import (
 	_ "github.com/containerd/aufs"
 	_ "github.com/containerd/containerd/metrics/cgroups"
-	_ "github.com/containerd/containerd/runtime/linux"
+	_ "github.com/containerd/containerd/runtime/v1/linux"
+	_ "github.com/containerd/containerd/runtime/v2"
 	_ "github.com/containerd/containerd/snapshots/native"
 	_ "github.com/containerd/containerd/snapshots/overlay"
 	_ "github.com/containerd/zfs"

cmd/ctr/commands/commands.go

@@ -105,7 +105,7 @@ var (
 		},
 		cli.StringFlag{
 			Name:  "runtime",
-			Usage: "runtime name (io.containerd.runtime.v1.linux, io.containerd.runtime.v1.windows, io.containerd.runtime.v1.com.vmware.linux)",
+			Usage: "runtime name",
 			Value: fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS),
 		},
 		cli.BoolFlag{

cmd/ctr/commands/shim/shim.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/containerd/console"
 	"github.com/containerd/containerd/cmd/ctr/commands"
-	shim "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v2/task"
 	"github.com/containerd/ttrpc"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -36,8 +36,6 @@ import (
 	"github.com/urfave/cli"
 )
 
-var empty = &ptypes.Empty{}
-
 var fifoFlags = []cli.Flag{
 	cli.StringFlag{
 		Name:  "stdin",
@@ -57,7 +55,7 @@ var fifoFlags = []cli.Flag{
 	},
 }
 
-// Command is the cli command for interacting with a shim
+// Command is the cli command for interacting with a task
 var Command = cli.Command{
 	Name:  "shim",
 	Usage: "interact with a shim directly",
@@ -77,13 +75,13 @@ var Command = cli.Command{
 
 var startCommand = cli.Command{
 	Name:  "start",
-	Usage: "start a container with a shim",
+	Usage: "start a container with a task",
 	Action: func(context *cli.Context) error {
-		service, err := getShimService(context)
+		service, err := getTaskService(context)
 		if err != nil {
 			return err
 		}
-		_, err = service.Start(gocontext.Background(), &shim.StartRequest{
+		_, err = service.Start(gocontext.Background(), &task.StartRequest{
 			ID: context.Args().First(),
 		})
 		return err
@@ -92,13 +90,15 @@ var startCommand = cli.Command{
 
 var deleteCommand = cli.Command{
 	Name:  "delete",
-	Usage: "delete a container with a shim",
+	Usage: "delete a container with a task",
 	Action: func(context *cli.Context) error {
-		service, err := getShimService(context)
+		service, err := getTaskService(context)
 		if err != nil {
 			return err
 		}
-		r, err := service.Delete(gocontext.Background(), empty)
+		r, err := service.Delete(gocontext.Background(), &task.DeleteRequest{
+			ID: context.Args().First(),
+		})
 		if err != nil {
 			return err
 		}
@@ -109,13 +109,13 @@ var deleteCommand = cli.Command{
 
 var stateCommand = cli.Command{
 	Name:  "state",
-	Usage: "get the state of all the processes of the shim",
+	Usage: "get the state of all the processes of the task",
 	Action: func(context *cli.Context) error {
-		service, err := getShimService(context)
+		service, err := getTaskService(context)
 		if err != nil {
 			return err
 		}
-		r, err := service.State(gocontext.Background(), &shim.StateRequest{
+		r, err := service.State(gocontext.Background(), &task.StateRequest{
 			ID: context.Args().First(),
 		})
 		if err != nil {
@@ -128,7 +128,7 @@ var stateCommand = cli.Command{
 
 var execCommand = cli.Command{
 	Name:  "exec",
-	Usage: "exec a new process in the shim's container",
+	Usage: "exec a new process in the task's container",
 	Flags: append(fifoFlags,
 		cli.BoolFlag{
 			Name:  "attach,a",
@@ -149,7 +149,7 @@ var execCommand = cli.Command{
 		},
 	),
 	Action: func(context *cli.Context) error {
-		service, err := getShimService(context)
+		service, err := getTaskService(context)
 		if err != nil {
 			return err
 		}
@@ -178,7 +178,7 @@ var execCommand = cli.Command{
 			return err
 		}
 
-		rq := &shim.ExecProcessRequest{
+		rq := &task.ExecProcessRequest{
 			ID: id,
 			Spec: &ptypes.Any{
 				TypeUrl: url,
@@ -192,7 +192,7 @@ var execCommand = cli.Command{
 		if _, err := service.Exec(ctx, rq); err != nil {
 			return err
 		}
-		r, err := service.Start(ctx, &shim.StartRequest{
+		r, err := service.Start(ctx, &task.StartRequest{
 			ID: id,
 		})
 		if err != nil {
@@ -211,7 +211,7 @@ var execCommand = cli.Command{
 				if err != nil {
 					return err
 				}
-				if _, err := service.ResizePty(ctx, &shim.ResizePtyRequest{
+				if _, err := service.ResizePty(ctx, &task.ResizePtyRequest{
 					ID:     id,
 					Width:  uint32(size.Width),
 					Height: uint32(size.Height),
@@ -225,7 +225,7 @@ var execCommand = cli.Command{
 	},
 }
 
-func getShimService(context *cli.Context) (shim.ShimService, error) {
+func getTaskService(context *cli.Context) (task.TaskService, error) {
 	bindSocket := context.GlobalString("socket")
 	if bindSocket == "" {
 		return nil, errors.New("socket path must be specified")
@@ -241,5 +241,5 @@ func getShimService(context *cli.Context) (shim.ShimService, error) {
 	// TODO(stevvooe): This actually leaks the connection. We were leaking it
 	// before, so may not be a huge deal.
 
-	return shim.NewShimClient(client), nil
+	return task.NewTaskClient(client), nil
 }

cmd/ctr/commands/tasks/checkpoint.go

@@ -18,9 +18,12 @@ package tasks
 
 import (
 	"fmt"
+	"runtime"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cmd/ctr/commands"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 )
@@ -34,6 +37,11 @@ var checkpointCommand = cli.Command{
 			Name:  "exit",
 			Usage: "stop the container after the checkpoint",
 		},
+		cli.StringFlag{
+			Name:  "runtime",
+			Usage: "runtime name",
+			Value: fmt.Sprintf("io.containerd.runtime.v1.%s", runtime.GOOS),
+		},
 	},
 	Action: func(context *cli.Context) error {
 		id := context.Args().First()
@@ -55,7 +63,7 @@ var checkpointCommand = cli.Command{
 		}
 		var opts []containerd.CheckpointTaskOpts
 		if context.Bool("exit") {
-			opts = append(opts, containerd.WithExit)
+			opts = append(opts, withExit(context.String("runtime")))
 		}
 		checkpoint, err := task.Checkpoint(ctx, opts...)
 		if err != nil {
@@ -65,3 +73,19 @@ var checkpointCommand = cli.Command{
 		return nil
 	},
 }
+
+func withExit(rt string) containerd.CheckpointTaskOpts {
+	return func(r *containerd.CheckpointTaskInfo) error {
+		switch rt {
+		case "io.containerd.runc.v1":
+			r.Options = &options.CheckpointOptions{
+				Exit: true,
+			}
+		default:
+			r.Options = &runctypes.CheckpointOptions{
+				Exit: true,
+			}
+		}
+		return nil
+	}
+}

container_checkpoint_test.go

@@ -28,6 +28,8 @@ import (
 	"testing"
 
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/runtime/linux/runctypes"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 )
 
 func TestCheckpointRestorePTY(t *testing.T) {
@@ -81,7 +83,7 @@ func TestCheckpointRestorePTY(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, WithExit)
+	checkpoint, err := task.Checkpoint(ctx, withExit(client))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -176,7 +178,7 @@ func TestCheckpointRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, WithExit)
+	checkpoint, err := task.Checkpoint(ctx, withExit(client))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -245,7 +247,7 @@ func TestCheckpointRestoreNewContainer(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	checkpoint, err := task.Checkpoint(ctx, WithExit)
+	checkpoint, err := task.Checkpoint(ctx, withExit(client))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -343,3 +345,19 @@ func TestCheckpointLeaveRunning(t *testing.T) {
 
 	<-statusC
 }
+
+func withExit(client *Client) CheckpointTaskOpts {
+	return func(r *CheckpointTaskInfo) error {
+		switch client.runtime {
+		case "io.containerd.runc.v1":
+			r.Options = &options.CheckpointOptions{
+				Exit: true,
+			}
+		default:
+			r.Options = &runctypes.CheckpointOptions{
+				Exit: true,
+			}
+		}
+		return nil
+	}
+}

container_linux_test.go

@@ -37,6 +37,7 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
@@ -129,6 +130,9 @@ func TestShimInCgroup(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer client.Close()
+	if client.runtime == "io.containerd.runc.v1" {
+		t.Skip()
+	}
 
 	var (
 		ctx, cancel = testContext()
@@ -871,70 +875,6 @@ func TestContainerKillAll(t *testing.T) {
 	}
 }
 
-func TestShimSigkilled(t *testing.T) {
-	client, err := newClient(t, address)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer client.Close()
-
-	var (
-		image       Image
-		ctx, cancel = testContext()
-		id          = t.Name()
-	)
-	defer cancel()
-
-	// redis unset its PDeathSignal making it a good candidate
-	image, err = client.Pull(ctx, "docker.io/library/redis:alpine", WithPullUnpack)
-	if err != nil {
-		t.Fatal(err)
-	}
-	container, err := client.NewContainer(ctx, id, WithNewSpec(oci.WithImageConfig(image)), WithNewSnapshot(id, image))
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer container.Delete(ctx, WithSnapshotCleanup)
-
-	task, err := container.NewTask(ctx, empty())
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer task.Delete(ctx)
-
-	statusC, err := task.Wait(ctx)
-	if err != nil {
-		t.Error(err)
-	}
-
-	pid := task.Pid()
-	if pid <= 0 {
-		t.Fatalf("invalid task pid %d", pid)
-	}
-
-	if err := task.Start(ctx); err != nil {
-		t.Fatal(err)
-	}
-
-	// SIGKILL the shim
-	if err := exec.Command("pkill", "-KILL", "containerd-s").Run(); err != nil {
-		t.Fatalf("failed to kill shim: %v", err)
-	}
-
-	<-statusC
-
-	for i := 0; i < 10; i++ {
-		if err := unix.Kill(int(pid), 0); err == unix.ESRCH {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if err := unix.Kill(int(pid), 0); err != unix.ESRCH {
-		t.Errorf("pid %d still exists", pid)
-	}
-
-}
-
 func TestDaemonRestartWithRunningShim(t *testing.T) {
 	client, err := newClient(t, address)
 	if err != nil {
@@ -1014,7 +954,7 @@ func TestDaemonRestartWithRunningShim(t *testing.T) {
 	}
 }
 
-func TestContainerRuntimeOptions(t *testing.T) {
+func TestContainerRuntimeOptionsv1(t *testing.T) {
 	t.Parallel()
 
 	client, err := newClient(t, address)
@@ -1057,6 +997,49 @@ func TestContainerRuntimeOptions(t *testing.T) {
 	}
 }
 
+func TestContainerRuntimeOptionsv2(t *testing.T) {
+	t.Parallel()
+
+	client, err := newClient(t, address)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	var (
+		image       Image
+		ctx, cancel = testContext()
+		id          = t.Name()
+	)
+	defer cancel()
+
+	image, err = client.GetImage(ctx, testImage)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	container, err := client.NewContainer(
+		ctx, id,
+		WithNewSpec(oci.WithImageConfig(image), withExitStatus(7)),
+		WithNewSnapshot(id, image),
+		WithRuntime("io.containerd.runc.v1", &options.Options{BinaryName: "no-runc"}),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer container.Delete(ctx, WithSnapshotCleanup)
+
+	task, err := container.NewTask(ctx, empty())
+	if err == nil {
+		t.Errorf("task creation should have failed")
+		task.Delete(ctx)
+		return
+	}
+	if !strings.Contains(err.Error(), `"no-runc"`) {
+		t.Errorf("task creation should have failed because of lack of executable. Instead failed with: %v", err.Error())
+	}
+}
+
 func TestContainerKillInitPidHost(t *testing.T) {
 	client, err := newClient(t, address)
 	if err != nil {
@@ -1183,11 +1166,21 @@ func testUserNamespaces(t *testing.T, readonlyRootFS bool) {
 	}
 	defer container.Delete(ctx, WithSnapshotCleanup)
 
-	task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio), func(_ context.Context, client *Client, r *TaskInfo) error {
-		r.Options = &runctypes.CreateOptions{
+	var copts interface{}
+	if client.runtime == "io.containerd.runc.v1" {
+		copts = &options.Options{
 			IoUid: 1000,
 			IoGid: 1000,
 		}
+	} else {
+		copts = &runctypes.CreateOptions{
+			IoUid: 1000,
+			IoGid: 1000,
+		}
+	}
+
+	task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio), func(_ context.Context, client *Client, r *TaskInfo) error {
+		r.Options = copts
 		return nil
 	})
 	if err != nil {

container_test.go

@@ -1239,7 +1239,7 @@ func TestContainerMetrics(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer task.Delete(ctx)
+	defer task.Delete(ctx, WithProcessKill)
 
 	statusC, err := task.Wait(ctx)
 	if err != nil {
@@ -1249,6 +1249,7 @@ func TestContainerMetrics(t *testing.T) {
 	metric, err := task.Metrics(ctx)
 	if err != nil {
 		t.Error(err)
+		return
 	}
 	if metric.ID != id {
 		t.Errorf("expected metric id %q but received %q", id, metric.ID)

daemon_config_linux_test.go

@@ -19,7 +19,6 @@ package containerd
 import (
 	"bytes"
 	"context"
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -30,6 +29,7 @@ import (
 
 	"github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/pkg/testutil"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/containerd/services/server"
 )
 
@@ -122,14 +122,11 @@ func testDaemonRuntimeRoot(t *testing.T, noShim bool) {
 			os.RemoveAll(runtimeRoot)
 		}
 	}()
-	configTOML := fmt.Sprintf(`
+	configTOML := `
 [plugins]
- [plugins.linux]
-   no_shim = %v
-   runtime_root = "%s"
  [plugins.cri]
    stream_server_port = "0"
-`, noShim, runtimeRoot)
+`
 
 	client, _, cleanup := newDaemonWithConfig(t, configTOML)
 	defer cleanup()
@@ -143,7 +140,9 @@ func testDaemonRuntimeRoot(t *testing.T, noShim bool) {
 	}
 
 	id := t.Name()
-	container, err := client.NewContainer(ctx, id, WithNewSpec(oci.WithImageConfig(image), withProcessArgs("top")), WithNewSnapshot(id, image))
+	container, err := client.NewContainer(ctx, id, WithNewSpec(oci.WithImageConfig(image), withProcessArgs("top")), WithNewSnapshot(id, image), WithRuntime("io.containerd.runc.v1", &options.Options{
+		Root: runtimeRoot,
+	}))
 	if err != nil {
 		t.Fatal(err)
 	}

metrics/cgroups/cgroups.go

@@ -23,13 +23,14 @@ import (
 
 	"github.com/containerd/cgroups"
 	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/linux"
+	"github.com/containerd/containerd/runtime/v1/linux"
 	metrics "github.com/docker/go-metrics"
 	"github.com/sirupsen/logrus"
 )
@@ -80,16 +81,21 @@ type cgroupsMonitor struct {
 }
 
 func (m *cgroupsMonitor) Monitor(c runtime.Task) error {
-	info := c.Info()
-	t := c.(*linux.Task)
+	if err := m.collector.Add(c); err != nil {
+		return err
+	}
+	t, ok := c.(*linux.Task)
+	if !ok {
+		return nil
+	}
 	cg, err := t.Cgroup()
 	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return nil
+		}
 		return err
 	}
-	if err := m.collector.Add(info.ID, info.Namespace, cg); err != nil {
-		return err
-	}
-	err = m.oom.Add(info.ID, info.Namespace, cg, m.trigger)
+	err = m.oom.Add(c.ID(), c.Namespace(), cg, m.trigger)
 	if err == cgroups.ErrMemoryNotSupported {
 		logrus.WithError(err).Warn("OOM monitoring failed")
 		return nil
@@ -98,17 +104,7 @@ func (m *cgroupsMonitor) Monitor(c runtime.Task) error {
 }
 
 func (m *cgroupsMonitor) Stop(c runtime.Task) error {
-	info := c.Info()
-	t := c.(*linux.Task)
-
-	cgroup, err := t.Cgroup()
-	if err != nil {
-		log.G(m.context).WithError(err).Warnf("unable to retrieve cgroup on stop")
-	} else {
-		m.collector.collect(info.ID, info.Namespace, cgroup, m.collector.storedMetrics, false, nil)
-	}
-
-	m.collector.Remove(info.ID, info.Namespace)
+	m.collector.Remove(c)
 	return nil
 }
 

metrics/cgroups/metrics.go

@@ -19,12 +19,16 @@
 package cgroups
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"sync"
 
 	"github.com/containerd/cgroups"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/runtime"
+	"github.com/containerd/typeurl"
 	metrics "github.com/docker/go-metrics"
 	"github.com/prometheus/client_golang/prometheus"
 )
@@ -49,7 +53,7 @@ func newCollector(ns *metrics.Namespace) *collector {
 	// add machine cpus and memory info
 	c := &collector{
 		ns:    ns,
-		cgroups: make(map[string]*task),
+		tasks: make(map[string]runtime.Task),
 	}
 	c.metrics = append(c.metrics, pidMetrics...)
 	c.metrics = append(c.metrics, cpuMetrics...)
@@ -61,12 +65,6 @@ func newCollector(ns *metrics.Namespace) *collector {
 	return c
 }
 
-type task struct {
-	id        string
-	namespace string
-	cgroup    cgroups.Cgroup
-}
-
 func taskID(id, namespace string) string {
 	return fmt.Sprintf("%s-%s", id, namespace)
 }
@@ -76,7 +74,7 @@ func taskID(id, namespace string) string {
 type collector struct {
 	mu sync.RWMutex
 
-	cgroups       map[string]*task
+	tasks         map[string]runtime.Task
 	ns            *metrics.Namespace
 	metrics       []*metric
 	storedMetrics chan prometheus.Metric
@@ -91,9 +89,9 @@ func (c *collector) Describe(ch chan<- *prometheus.Desc) {
 func (c *collector) Collect(ch chan<- prometheus.Metric) {
 	c.mu.RLock()
 	wg := &sync.WaitGroup{}
-	for _, t := range c.cgroups {
+	for _, t := range c.tasks {
 		wg.Add(1)
-		go c.collect(t.id, t.namespace, t.cgroup, ch, true, wg)
+		go c.collect(t, ch, true, wg)
 	}
 storedLoop:
 	for {
@@ -109,45 +107,52 @@ storedLoop:
 	wg.Wait()
 }
 
-func (c *collector) collect(id, namespace string, cg cgroups.Cgroup, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
+func (c *collector) collect(t runtime.Task, ch chan<- prometheus.Metric, block bool, wg *sync.WaitGroup) {
 	if wg != nil {
 		defer wg.Done()
 	}
-
-	stats, err := cg.Stat(cgroups.IgnoreNotExist)
+	ctx := namespaces.WithNamespace(context.Background(), t.Namespace())
+	stats, err := t.Stats(ctx)
 	if err != nil {
-		log.L.WithError(err).Errorf("stat cgroup %s", id)
+		log.L.WithError(err).Errorf("stat task %s", t.ID())
+		return
+	}
+	data, err := typeurl.UnmarshalAny(stats)
+	if err != nil {
+		log.L.WithError(err).Errorf("unmarshal stats for %s", t.ID())
+		return
+	}
+	s, ok := data.(*cgroups.Metrics)
+	if !ok {
+		log.L.WithError(err).Errorf("invalid metric type for %s", t.ID())
 		return
 	}
 	for _, m := range c.metrics {
-		m.collect(id, namespace, stats, c.ns, ch, block)
+		m.collect(t.ID(), t.Namespace(), s, c.ns, ch, block)
 	}
 }
 
 // Add adds the provided cgroup and id so that metrics are collected and exported
-func (c *collector) Add(id, namespace string, cg cgroups.Cgroup) error {
+func (c *collector) Add(t runtime.Task) error {
 	if c.ns == nil {
 		return nil
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	if _, ok := c.cgroups[taskID(id, namespace)]; ok {
+	id := taskID(t.ID(), t.Namespace())
+	if _, ok := c.tasks[id]; ok {
 		return ErrAlreadyCollected
 	}
-	c.cgroups[taskID(id, namespace)] = &task{
-		id:        id,
-		namespace: namespace,
-		cgroup:    cg,
-	}
+	c.tasks[id] = t
 	return nil
 }
 
 // Remove removes the provided cgroup by id from the collector
-func (c *collector) Remove(id, namespace string) {
+func (c *collector) Remove(t runtime.Task) {
 	if c.ns == nil {
 		return
 	}
 	c.mu.Lock()
 	defer c.mu.Unlock()
-	delete(c.cgroups, taskID(id, namespace))
+	delete(c.tasks, taskID(t.ID(), t.Namespace()))
 }

plugin/plugin.go

@@ -58,6 +58,8 @@ const (
 	InternalPlugin Type = "io.containerd.internal.v1"
 	// RuntimePlugin implements a runtime
 	RuntimePlugin Type = "io.containerd.runtime.v1"
+	// RuntimePluginV2 implements a runtime v2
+	RuntimePluginV2 Type = "io.containerd.runtime.v2"
 	// ServicePlugin implements a internal service
 	ServicePlugin Type = "io.containerd.service.v1"
 	// GRPCPlugin implements a grpc service

runtime/runtime.go

@@ -42,8 +42,12 @@ type CreateOpts struct {
 	IO IO
 	// Checkpoint digest to restore container state
 	Checkpoint string
-	// Options for the runtime and container
-	Options *types.Any
+	// RuntimeOptions for the runtime
+	RuntimeOptions *types.Any
+	// TaskOptions received for the task
+	TaskOptions *types.Any
+	// Runtime to use
+	Runtime string
 }
 
 // Exit information for a process
@@ -64,7 +68,5 @@ type PlatformRuntime interface {
 	Get(context.Context, string) (Task, error)
 	// Tasks returns all the current tasks for the runtime.
 	// Any container runs at most one task at a time.
-	Tasks(context.Context) ([]Task, error)
-	// Delete removes the task in the runtime.
-	Delete(context.Context, Task) (*Exit, error)
+	Tasks(context.Context, bool) ([]Task, error)
 }

runtime/task.go

@@ -46,14 +46,16 @@ type Process interface {
 	Start(context.Context) error
 	// Wait for the process to exit
 	Wait(context.Context) (*Exit, error)
+	// Delete deletes the process
+	Delete(context.Context) (*Exit, error)
 }
 
 // Task is the runtime object for an executing container
 type Task interface {
 	Process
 
-	// Information of the container
-	Info() TaskInfo
+	// Namespace that the task exists in
+	Namespace() string
 	// Pause pauses the container process
 	Pause(context.Context) error
 	// Resume unpauses the container process
@@ -64,14 +66,12 @@ type Task interface {
 	Pids(context.Context) ([]ProcessInfo, error)
 	// Checkpoint checkpoints a container to an image with live system data
 	Checkpoint(context.Context, string, *types.Any) error
-	// DeleteProcess deletes a specific exec process via its id
-	DeleteProcess(context.Context, string) (*Exit, error)
 	// Update sets the provided resources to a running task
 	Update(context.Context, *types.Any) error
 	// Process returns a process within the task for the provided id
 	Process(context.Context, string) (Process, error)
-	// Metrics returns runtime specific metrics for a task
-	Metrics(context.Context) (interface{}, error)
+	// Stats returns runtime specific metrics for a task
+	Stats(context.Context) (*types.Any, error)
 }
 
 // ExecOpts provides additional options for additional processes running in a task

runtime/task_list.go

@@ -64,14 +64,22 @@ func (l *TaskList) Get(ctx context.Context, id string) (Task, error) {
 }
 
 // GetAll tasks under a namespace
-func (l *TaskList) GetAll(ctx context.Context) ([]Task, error) {
+func (l *TaskList) GetAll(ctx context.Context, noNS bool) ([]Task, error) {
 	l.mu.Lock()
 	defer l.mu.Unlock()
+	var o []Task
+	if noNS {
+		for ns := range l.tasks {
+			for _, t := range l.tasks[ns] {
+				o = append(o, t)
+			}
+		}
+		return o, nil
+	}
 	namespace, err := namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return nil, err
 	}
-	var o []Task
 	tasks, ok := l.tasks[namespace]
 	if !ok {
 		return o, nil

runtime/v1/linux/bundle.go

@@ -26,8 +26,8 @@ import (
 
 	"github.com/containerd/containerd/events/exchange"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
-	"github.com/containerd/containerd/runtime/shim"
-	"github.com/containerd/containerd/runtime/shim/client"
+	"github.com/containerd/containerd/runtime/v1/shim"
+	"github.com/containerd/containerd/runtime/v1/shim/client"
 	"github.com/pkg/errors"
 )
 

runtime/v1/linux/proc/exec.go

@@ -89,7 +89,7 @@ func (e *execProcess) ExitedAt() time.Time {
 func (e *execProcess) setExited(status int) {
 	e.status = status
 	e.exited = time.Now()
-	e.parent.platform.ShutdownConsole(context.Background(), e.console)
+	e.parent.Platform.ShutdownConsole(context.Background(), e.console)
 	close(e.waitBlock)
 }
 
@@ -180,7 +180,7 @@ func (e *execProcess) start(ctx context.Context) (err error) {
 		if err != nil {
 			return errors.Wrap(err, "failed to retrieve console master")
 		}
-		if e.console, err = e.parent.platform.CopyConsole(ctx, console, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, &copyWaitGroup); err != nil {
+		if e.console, err = e.parent.Platform.CopyConsole(ctx, console, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg, &copyWaitGroup); err != nil {
 			return errors.Wrap(err, "failed to start console copy")
 		}
 	} else if !e.stdio.IsNull() {

runtime/v1/linux/proc/init.go

@@ -33,11 +33,9 @@ import (
 	"github.com/containerd/console"
 	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
-	"github.com/containerd/containerd/runtime/linux/runctypes"
 	"github.com/containerd/containerd/runtime/proc"
 	"github.com/containerd/fifo"
 	runc "github.com/containerd/go-runc"
-	"github.com/containerd/typeurl"
 	google_protobuf "github.com/gogo/protobuf/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -59,12 +57,12 @@ type Init struct {
 
 	waitBlock chan struct{}
 
-	workDir string
+	WorkDir string
 
 	id           string
-	bundle   string
+	Bundle       string
 	console      console.Console
-	platform proc.Platform
+	Platform     proc.Platform
 	io           runc.IO
 	runtime      *runc.Runc
 	status       int
@@ -73,9 +71,11 @@ type Init struct {
 	closers      []io.Closer
 	stdin        io.Closer
 	stdio        proc.Stdio
-	rootfs   string
+	Rootfs       string
 	IoUID        int
 	IoGID        int
+	NoPivotRoot  bool
+	NoNewKeyring bool
 }
 
 // NewRunc returns a new runc instance for a process
@@ -94,90 +94,50 @@ func NewRunc(root, path, namespace, runtime, criu string, systemd bool) *runc.Ru
 	}
 }
 
-// New returns a new init process
-func New(context context.Context, path, workDir, runtimeRoot, namespace, criu string, systemdCgroup bool, platform proc.Platform, r *CreateConfig) (*Init, error) {
-	var success bool
-
-	var options runctypes.CreateOptions
-	if r.Options != nil {
-		v, err := typeurl.UnmarshalAny(r.Options)
-		if err != nil {
-			return nil, err
-		}
-		options = *v.(*runctypes.CreateOptions)
-	}
-
-	rootfs := filepath.Join(path, "rootfs")
-	// count the number of successful mounts so we can undo
-	// what was actually done rather than what should have been
-	// done.
-	defer func() {
-		if success {
-			return
-		}
-		if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
-			log.G(context).WithError(err2).Warn("Failed to cleanup rootfs mount")
-		}
-	}()
-	for _, rm := range r.Rootfs {
-		m := &mount.Mount{
-			Type:    rm.Type,
-			Source:  rm.Source,
-			Options: rm.Options,
-		}
-		if err := m.Mount(rootfs); err != nil {
-			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
-		}
-	}
-	runtime := NewRunc(runtimeRoot, path, namespace, r.Runtime, criu, systemdCgroup)
+// New returns a new process
+func New(id string, runtime *runc.Runc, stdio proc.Stdio) *Init {
 	p := &Init{
-		id:       r.ID,
-		bundle:   r.Bundle,
+		id:        id,
 		runtime:   runtime,
-		platform: platform,
-		stdio: proc.Stdio{
-			Stdin:    r.Stdin,
-			Stdout:   r.Stdout,
-			Stderr:   r.Stderr,
-			Terminal: r.Terminal,
-		},
-		rootfs:    rootfs,
-		workDir:   workDir,
+		stdio:     stdio,
 		status:    0,
 		waitBlock: make(chan struct{}),
-		IoUID:     int(options.IoUid),
-		IoGID:     int(options.IoGid),
 	}
 	p.initState = &createdState{p: p}
+	return p
+}
+
+// Create the process with the provided config
+func (p *Init) Create(ctx context.Context, r *CreateConfig) error {
 	var (
 		err    error
 		socket *runc.Socket
 	)
 	if r.Terminal {
 		if socket, err = runc.NewTempConsoleSocket(); err != nil {
-			return nil, errors.Wrap(err, "failed to create OCI runtime console socket")
+			return errors.Wrap(err, "failed to create OCI runtime console socket")
 		}
 		defer socket.Close()
 	} else if hasNoIO(r) {
 		if p.io, err = runc.NewNullIO(); err != nil {
-			return nil, errors.Wrap(err, "creating new NULL IO")
+			return errors.Wrap(err, "creating new NULL IO")
 		}
 	} else {
-		if p.io, err = runc.NewPipeIO(int(options.IoUid), int(options.IoGid)); err != nil {
-			return nil, errors.Wrap(err, "failed to create OCI runtime io pipes")
+		if p.io, err = runc.NewPipeIO(p.IoUID, p.IoGID); err != nil {
+			return errors.Wrap(err, "failed to create OCI runtime io pipes")
 		}
 	}
-	pidFile := filepath.Join(path, InitPidFile)
+	pidFile := filepath.Join(p.Bundle, InitPidFile)
 	if r.Checkpoint != "" {
 		opts := &runc.RestoreOpts{
 			CheckpointOpts: runc.CheckpointOpts{
 				ImagePath:  r.Checkpoint,
-				WorkDir:    p.workDir,
+				WorkDir:    p.WorkDir,
 				ParentPath: r.ParentCheckpoint,
 			},
 			PidFile:     pidFile,
 			IO:          p.io,
-			NoPivot:     options.NoPivotRoot,
+			NoPivot:     p.NoPivotRoot,
 			Detach:      true,
 			NoSubreaper: true,
 		}
@@ -185,25 +145,24 @@ func New(context context.Context, path, workDir, runtimeRoot, namespace, criu st
 			p:    p,
 			opts: opts,
 		}
-		success = true
-		return p, nil
+		return nil
 	}
 	opts := &runc.CreateOpts{
 		PidFile:      pidFile,
 		IO:           p.io,
-		NoPivot:      options.NoPivotRoot,
-		NoNewKeyring: options.NoNewKeyring,
+		NoPivot:      p.NoPivotRoot,
+		NoNewKeyring: p.NoNewKeyring,
 	}
 	if socket != nil {
 		opts.ConsoleSocket = socket
 	}
-	if err := p.runtime.Create(context, r.ID, r.Bundle, opts); err != nil {
-		return nil, p.runtimeError(err, "OCI runtime create failed")
+	if err := p.runtime.Create(ctx, r.ID, r.Bundle, opts); err != nil {
+		return p.runtimeError(err, "OCI runtime create failed")
 	}
 	if r.Stdin != "" {
-		sc, err := fifo.OpenFifo(context, r.Stdin, syscall.O_WRONLY|syscall.O_NONBLOCK, 0)
+		sc, err := fifo.OpenFifo(ctx, r.Stdin, syscall.O_WRONLY|syscall.O_NONBLOCK, 0)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to open stdin fifo %s", r.Stdin)
+			return errors.Wrapf(err, "failed to open stdin fifo %s", r.Stdin)
 		}
 		p.stdin = sc
 		p.closers = append(p.closers, sc)
@@ -212,27 +171,26 @@ func New(context context.Context, path, workDir, runtimeRoot, namespace, criu st
 	if socket != nil {
 		console, err := socket.ReceiveMaster()
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to retrieve console master")
+			return errors.Wrap(err, "failed to retrieve console master")
 		}
-		console, err = platform.CopyConsole(context, console, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup)
+		console, err = p.Platform.CopyConsole(ctx, console, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup)
 		if err != nil {
-			return nil, errors.Wrap(err, "failed to start console copy")
+			return errors.Wrap(err, "failed to start console copy")
 		}
 		p.console = console
 	} else if !hasNoIO(r) {
-		if err := copyPipes(context, p.io, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup); err != nil {
-			return nil, errors.Wrap(err, "failed to start io pipe copy")
+		if err := copyPipes(ctx, p.io, r.Stdin, r.Stdout, r.Stderr, &p.wg, &copyWaitGroup); err != nil {
+			return errors.Wrap(err, "failed to start io pipe copy")
 		}
 	}
 
 	copyWaitGroup.Wait()
 	pid, err := runc.ReadPidFile(pidFile)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to retrieve OCI runtime container pid")
+		return errors.Wrap(err, "failed to retrieve OCI runtime container pid")
 	}
 	p.pid = pid
-	success = true
-	return p, nil
+	return nil
 }
 
 // Wait for the process to exit
@@ -286,7 +244,7 @@ func (p *Init) start(context context.Context) error {
 func (p *Init) setExited(status int) {
 	p.exited = time.Now()
 	p.status = status
-	p.platform.ShutdownConsole(context.Background(), p.console)
+	p.Platform.ShutdownConsole(context.Background(), p.console)
 	close(p.waitBlock)
 }
 
@@ -312,7 +270,7 @@ func (p *Init) delete(context context.Context) error {
 		}
 		p.io.Close()
 	}
-	if err2 := mount.UnmountAll(p.rootfs, 0); err2 != nil {
+	if err2 := mount.UnmountAll(p.Rootfs, 0); err2 != nil {
 		log.G(context).WithError(err2).Warn("failed to cleanup rootfs mount")
 		if err == nil {
 			err = errors.Wrap(err2, "failed rootfs umount")
@@ -390,30 +348,22 @@ func (p *Init) exec(context context.Context, path string, r *ExecConfig) (proc.P
 }
 
 func (p *Init) checkpoint(context context.Context, r *CheckpointConfig) error {
-	var options runctypes.CheckpointOptions
-	if r.Options != nil {
-		v, err := typeurl.UnmarshalAny(r.Options)
-		if err != nil {
-			return err
-		}
-		options = *v.(*runctypes.CheckpointOptions)
-	}
 	var actions []runc.CheckpointAction
-	if !options.Exit {
+	if !r.Exit {
 		actions = append(actions, runc.LeaveRunning)
 	}
-	work := filepath.Join(p.workDir, "criu-work")
+	work := filepath.Join(p.WorkDir, "criu-work")
 	defer os.RemoveAll(work)
 	if err := p.runtime.Checkpoint(context, p.id, &runc.CheckpointOpts{
 		WorkDir:                  work,
 		ImagePath:                r.Path,
-		AllowOpenTCP:             options.OpenTcp,
-		AllowExternalUnixSockets: options.ExternalUnixSockets,
-		AllowTerminal:            options.Terminal,
-		FileLocks:                options.FileLocks,
-		EmptyNamespaces:          options.EmptyNamespaces,
+		AllowOpenTCP:             r.AllowOpenTCP,
+		AllowExternalUnixSockets: r.AllowExternalUnixSockets,
+		AllowTerminal:            r.AllowTerminal,
+		FileLocks:                r.FileLocks,
+		EmptyNamespaces:          r.EmptyNamespaces,
 	}, actions...); err != nil {
-		dumpLog := filepath.Join(p.bundle, "criu-dump.log")
+		dumpLog := filepath.Join(p.Bundle, "criu-dump.log")
 		if cerr := copyFile(dumpLog, filepath.Join(work, "dump.log")); cerr != nil {
 			log.G(context).Error(err)
 		}

runtime/v1/linux/proc/init_state.go

@@ -209,7 +209,7 @@ func (s *createdCheckpointState) Start(ctx context.Context) error {
 		s.opts.ConsoleSocket = socket
 	}
 
-	if _, err := s.p.runtime.Restore(ctx, p.id, p.bundle, s.opts); err != nil {
+	if _, err := s.p.runtime.Restore(ctx, p.id, p.Bundle, s.opts); err != nil {
 		return p.runtimeError(err, "OCI runtime restore failed")
 	}
 	if sio.Stdin != "" {
@@ -226,7 +226,7 @@ func (s *createdCheckpointState) Start(ctx context.Context) error {
 		if err != nil {
 			return errors.Wrap(err, "failed to retrieve console master")
 		}
-		console, err = p.platform.CopyConsole(ctx, console, sio.Stdin, sio.Stdout, sio.Stderr, &p.wg, &copyWaitGroup)
+		console, err = p.Platform.CopyConsole(ctx, console, sio.Stdin, sio.Stdout, sio.Stderr, &p.wg, &copyWaitGroup)
 		if err != nil {
 			return errors.Wrap(err, "failed to start console copy")
 		}

runtime/v1/linux/proc/types.go

@@ -56,5 +56,10 @@ type ExecConfig struct {
 // CheckpointConfig holds task checkpoint configuration
 type CheckpointConfig struct {
 	Path                     string
-	Options *google_protobuf.Any
+	Exit                     bool
+	AllowOpenTCP             bool
+	AllowExternalUnixSockets bool
+	AllowTerminal            bool
+	FileLocks                bool
+	EmptyNamespaces          []string
 }

runtime/v1/linux/process.go

@@ -25,7 +25,7 @@ import (
 	"github.com/containerd/containerd/api/types/task"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	shim "github.com/containerd/containerd/runtime/shim/v1"
+	shim "github.com/containerd/containerd/runtime/v1/shim/v1"
 	"github.com/containerd/ttrpc"
 	"github.com/pkg/errors"
 )
@@ -149,3 +149,18 @@ func (p *Process) Wait(ctx context.Context) (*runtime.Exit, error) {
 		Status:    r.ExitStatus,
 	}, nil
 }
+
+// Delete the process and return the exit status
+func (p *Process) Delete(ctx context.Context) (*runtime.Exit, error) {
+	r, err := p.t.shim.DeleteProcess(ctx, &shim.DeleteProcessRequest{
+		ID: p.id,
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return &runtime.Exit{
+		Status:    r.ExitStatus,
+		Timestamp: r.ExitedAt,
+		Pid:       r.Pid,
+	}, nil
+}

runtime/v1/linux/runtime.go

@@ -40,9 +40,9 @@ import (
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/linux/proc"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
-	shim "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v1/linux/proc"
+	shim "github.com/containerd/containerd/runtime/v1/shim/v1"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -69,7 +69,6 @@ func init() {
 		ID:     "linux",
 		InitFn: New,
 		Requires: []plugin.Type{
-			plugin.TaskMonitorPlugin,
 			plugin.MetadataPlugin,
 		},
 		Config: &Config{
@@ -105,10 +104,6 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 	if err := os.MkdirAll(ic.State, 0711); err != nil {
 		return nil, err
 	}
-	monitor, err := ic.Get(plugin.TaskMonitorPlugin)
-	if err != nil {
-		return nil, err
-	}
 	m, err := ic.Get(plugin.MetadataPlugin)
 	if err != nil {
 		return nil, err
@@ -117,7 +112,6 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 	r := &Runtime{
 		root:    ic.Root,
 		state:   ic.State,
-		monitor: monitor.(runtime.TaskMonitor),
 		tasks:   runtime.NewTaskList(),
 		db:      m.(*metadata.DB),
 		address: ic.Address,
@@ -128,8 +122,6 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-
-	// TODO: need to add the tasks to the monitor
 	for _, t := range tasks {
 		if err := r.tasks.AddWithNamespace(t.namespace, t); err != nil {
 			return nil, err
@@ -144,7 +136,6 @@ type Runtime struct {
 	state   string
 	address string
 
-	monitor runtime.TaskMonitor
 	tasks  *runtime.TaskList
 	db     *metadata.DB
 	events *exchange.Exchange
@@ -189,8 +180,8 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 	shimopt := ShimLocal(r.config, r.events)
 	if !r.config.NoShim {
 		var cgroup string
-		if opts.Options != nil {
-			v, err := typeurl.UnmarshalAny(opts.Options)
+		if opts.TaskOptions != nil {
+			v, err := typeurl.UnmarshalAny(opts.TaskOptions)
 			if err != nil {
 				return nil, err
 			}
@@ -205,14 +196,6 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 			}
 			lc := t.(*Task)
 
-			// Stop the monitor
-			if err := r.monitor.Stop(lc); err != nil {
-				log.G(ctx).WithError(err).WithFields(logrus.Fields{
-					"id":        id,
-					"namespace": namespace,
-				}).Warn("failed to stop monitor")
-			}
-
 			log.G(ctx).WithFields(logrus.Fields{
 				"id":        id,
 				"namespace": namespace,
@@ -252,7 +235,7 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 		Stderr:     opts.IO.Stderr,
 		Terminal:   opts.IO.Terminal,
 		Checkpoint: opts.Checkpoint,
-		Options:    opts.Options,
+		Options:    opts.TaskOptions,
 	}
 	for _, m := range opts.Rootfs {
 		sopts.Rootfs = append(sopts.Rootfs, &types.Mount{
@@ -265,24 +248,14 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 	if err != nil {
 		return nil, errdefs.FromGRPC(err)
 	}
-	t, err := newTask(id, namespace, int(cr.Pid), s, r.monitor, r.events,
-		proc.NewRunc(ropts.RuntimeRoot, sopts.Bundle, namespace, rt, ropts.CriuPath, ropts.SystemdCgroup))
+	t, err := newTask(id, namespace, int(cr.Pid), s, r.events,
+		proc.NewRunc(ropts.RuntimeRoot, sopts.Bundle, namespace, rt, ropts.CriuPath, ropts.SystemdCgroup), r.tasks, bundle)
 	if err != nil {
 		return nil, err
 	}
 	if err := r.tasks.Add(ctx, t); err != nil {
 		return nil, err
 	}
-	// after the task is created, add it to the monitor if it has a cgroup
-	// this can be different on a checkpoint/restore
-	if t.cg != nil {
-		if err = r.monitor.Monitor(t); err != nil {
-			if _, err := r.Delete(ctx, t); err != nil {
-				log.G(ctx).WithError(err).Error("deleting task after failed monitor")
-			}
-			return nil, err
-		}
-	}
 	r.events.Publish(ctx, runtime.TaskCreateEventTopic, &eventstypes.TaskCreate{
 		ContainerID: sopts.ID,
 		Bundle:      sopts.Bundle,
@@ -300,56 +273,9 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts
 	return t, nil
 }
 
-// Delete a task removing all on disk state
-func (r *Runtime) Delete(ctx context.Context, c runtime.Task) (*runtime.Exit, error) {
-	namespace, err := namespaces.NamespaceRequired(ctx)
-	if err != nil {
-		return nil, err
-	}
-	lc, ok := c.(*Task)
-	if !ok {
-		return nil, fmt.Errorf("task cannot be cast as *linux.Task")
-	}
-	if err := r.monitor.Stop(lc); err != nil {
-		return nil, err
-	}
-	bundle := loadBundle(
-		lc.id,
-		filepath.Join(r.state, namespace, lc.id),
-		filepath.Join(r.root, namespace, lc.id),
-	)
-
-	rsp, err := lc.shim.Delete(ctx, empty)
-	if err != nil {
-		if cerr := r.cleanupAfterDeadShim(ctx, bundle, namespace, c.ID(), lc.pid); cerr != nil {
-			log.G(ctx).WithError(err).Error("unable to cleanup task")
-		}
-		return nil, errdefs.FromGRPC(err)
-	}
-	r.tasks.Delete(ctx, lc.id)
-	if err := lc.shim.KillShim(ctx); err != nil {
-		log.G(ctx).WithError(err).Error("failed to kill shim")
-	}
-
-	if err := bundle.Delete(); err != nil {
-		log.G(ctx).WithError(err).Error("failed to delete bundle")
-	}
-	r.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
-		ContainerID: lc.id,
-		ExitStatus:  rsp.ExitStatus,
-		ExitedAt:    rsp.ExitedAt,
-		Pid:         rsp.Pid,
-	})
-	return &runtime.Exit{
-		Status:    rsp.ExitStatus,
-		Timestamp: rsp.ExitedAt,
-		Pid:       rsp.Pid,
-	}, nil
-}
-
 // Tasks returns all tasks known to the runtime
-func (r *Runtime) Tasks(ctx context.Context) ([]runtime.Task, error) {
-	return r.tasks.GetAll(ctx)
+func (r *Runtime) Tasks(ctx context.Context, all bool) ([]runtime.Task, error) {
+	return r.tasks.GetAll(ctx, all)
 }
 
 func (r *Runtime) restoreTasks(ctx context.Context) ([]*Task, error) {
@@ -422,8 +348,8 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
 			continue
 		}
 
-		t, err := newTask(id, ns, pid, s, r.monitor, r.events,
-			proc.NewRunc(ropts.RuntimeRoot, bundle.path, ns, ropts.Runtime, ropts.CriuPath, ropts.SystemdCgroup))
+		t, err := newTask(id, ns, pid, s, r.events,
+			proc.NewRunc(ropts.RuntimeRoot, bundle.path, ns, ropts.Runtime, ropts.CriuPath, ropts.SystemdCgroup), r.tasks, bundle)
 		if err != nil {
 			log.G(ctx).WithError(err).Error("loading task type")
 			continue

runtime/v1/linux/task.go

@@ -28,11 +28,13 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/events/exchange"
 	"github.com/containerd/containerd/identifiers"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/shim/client"
-	shim "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v1/shim/client"
+	shim "github.com/containerd/containerd/runtime/v1/shim/v1"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/ttrpc"
+	"github.com/containerd/typeurl"
 	"github.com/gogo/protobuf/types"
 	"github.com/pkg/errors"
 )
@@ -45,12 +47,12 @@ type Task struct {
 	shim      *client.Client
 	namespace string
 	cg        cgroups.Cgroup
-	monitor   runtime.TaskMonitor
 	events    *exchange.Exchange
-	runtime   *runc.Runc
+	tasks     *runtime.TaskList
+	bundle    *bundle
 }
 
-func newTask(id, namespace string, pid int, shim *client.Client, monitor runtime.TaskMonitor, events *exchange.Exchange, runtime *runc.Runc) (*Task, error) {
+func newTask(id, namespace string, pid int, shim *client.Client, events *exchange.Exchange, runtime *runc.Runc, list *runtime.TaskList, bundle *bundle) (*Task, error) {
 	var (
 		err error
 		cg  cgroups.Cgroup
@@ -67,9 +69,9 @@ func newTask(id, namespace string, pid int, shim *client.Client, monitor runtime
 		shim:      shim,
 		namespace: namespace,
 		cg:        cg,
-		monitor:   monitor,
 		events:    events,
-		runtime:   runtime,
+		tasks:     list,
+		bundle:    bundle,
 	}, nil
 }
 
@@ -78,13 +80,35 @@ func (t *Task) ID() string {
 	return t.id
 }
 
-// Info returns task information about the runtime and namespace
-func (t *Task) Info() runtime.TaskInfo {
-	return runtime.TaskInfo{
-		ID:        t.id,
-		Runtime:   pluginID,
-		Namespace: t.namespace,
+// Namespace of the task
+func (t *Task) Namespace() string {
+	return t.namespace
+}
+
+// Delete the task and return the exit status
+func (t *Task) Delete(ctx context.Context) (*runtime.Exit, error) {
+	rsp, err := t.shim.Delete(ctx, empty)
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
 	}
+	t.tasks.Delete(ctx, t.id)
+	if err := t.shim.KillShim(ctx); err != nil {
+		log.G(ctx).WithError(err).Error("failed to kill shim")
+	}
+	if err := t.bundle.Delete(); err != nil {
+		log.G(ctx).WithError(err).Error("failed to delete bundle")
+	}
+	t.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
+		ContainerID: t.id,
+		ExitStatus:  rsp.ExitStatus,
+		ExitedAt:    rsp.ExitedAt,
+		Pid:         rsp.Pid,
+	})
+	return &runtime.Exit{
+		Status:    rsp.ExitStatus,
+		Timestamp: rsp.ExitedAt,
+		Pid:       rsp.Pid,
+	}, nil
 }
 
 // Start the task
@@ -107,9 +131,6 @@ func (t *Task) Start(ctx context.Context) error {
 		t.mu.Lock()
 		t.cg = cg
 		t.mu.Unlock()
-		if err := t.monitor.Monitor(t); err != nil {
-			return err
-		}
 	}
 	t.events.Publish(ctx, runtime.TaskStartEventTopic, &eventstypes.TaskStart{
 		ContainerID: t.id,
@@ -270,21 +291,6 @@ func (t *Task) Checkpoint(ctx context.Context, path string, options *types.Any)
 	return nil
 }
 
-// DeleteProcess removes the provided process from the task and deletes all on disk state
-func (t *Task) DeleteProcess(ctx context.Context, id string) (*runtime.Exit, error) {
-	r, err := t.shim.DeleteProcess(ctx, &shim.DeleteProcessRequest{
-		ID: id,
-	})
-	if err != nil {
-		return nil, errdefs.FromGRPC(err)
-	}
-	return &runtime.Exit{
-		Status:    r.ExitStatus,
-		Timestamp: r.ExitedAt,
-		Pid:       r.Pid,
-	}, nil
-}
-
 // Update changes runtime information of a running task
 func (t *Task) Update(ctx context.Context, resources *types.Any) error {
 	if _, err := t.shim.Update(ctx, &shim.UpdateTaskRequest{
@@ -307,8 +313,8 @@ func (t *Task) Process(ctx context.Context, id string) (runtime.Process, error)
 	return p, nil
 }
 
-// Metrics returns runtime specific system level metric information for the task
-func (t *Task) Metrics(ctx context.Context) (interface{}, error) {
+// Stats returns runtime specific system level metric information for the task
+func (t *Task) Stats(ctx context.Context) (*types.Any, error) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 	if t.cg == nil {
@@ -318,7 +324,7 @@ func (t *Task) Metrics(ctx context.Context) (interface{}, error) {
 	if err != nil {
 		return nil, err
 	}
-	return stats, nil
+	return typeurl.MarshalAny(stats)
 }
 
 // Cgroup returns the underlying cgroup for a linux task

runtime/v1/shim/client/client.go

@@ -37,8 +37,8 @@ import (
 
 	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/log"
-	"github.com/containerd/containerd/runtime/shim"
-	shimapi "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v1/shim"
+	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
 	"github.com/containerd/containerd/sys"
 	ptypes "github.com/gogo/protobuf/types"
 )

runtime/v1/shim/local.go

@@ -23,7 +23,7 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/mount"
-	shimapi "github.com/containerd/containerd/runtime/shim/v1"
+	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
 	ptypes "github.com/gogo/protobuf/types"
 )
 

runtime/v1/shim/service.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"sync"
 
 	"github.com/containerd/console"
@@ -30,12 +31,13 @@ import (
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
 	"github.com/containerd/containerd/runtime"
-	"github.com/containerd/containerd/runtime/linux/proc"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
 	rproc "github.com/containerd/containerd/runtime/proc"
-	shimapi "github.com/containerd/containerd/runtime/shim/v1"
+	"github.com/containerd/containerd/runtime/v1/linux/proc"
+	shimapi "github.com/containerd/containerd/runtime/v1/shim/v1"
 	runc "github.com/containerd/go-runc"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -108,7 +110,7 @@ type Service struct {
 }
 
 // Create a new initial process and container with the underlying OCI runtime
-func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*shimapi.CreateTaskResponse, error) {
+func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ *shimapi.CreateTaskResponse, err error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -121,16 +123,8 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*sh
 			Options: m.Options,
 		})
 	}
-	process, err := proc.New(
-		ctx,
-		s.config.Path,
-		s.config.WorkDir,
-		s.config.RuntimeRoot,
-		s.config.Namespace,
-		s.config.Criu,
-		s.config.SystemdCgroup,
-		s.platform,
-		&proc.CreateConfig{
+
+	config := &proc.CreateConfig{
 		ID:               r.ID,
 		Bundle:           r.Bundle,
 		Runtime:          r.Runtime,
@@ -142,11 +136,42 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (*sh
 		Checkpoint:       r.Checkpoint,
 		ParentCheckpoint: r.ParentCheckpoint,
 		Options:          r.Options,
-		},
+	}
+	rootfs := filepath.Join(r.Bundle, "rootfs")
+	defer func() {
+		if err != nil {
+			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+				log.G(ctx).WithError(err2).Warn("Failed to cleanup rootfs mount")
+			}
+		}
+	}()
+	for _, rm := range mounts {
+		m := &mount.Mount{
+			Type:    rm.Type,
+			Source:  rm.Source,
+			Options: rm.Options,
+		}
+		if err := m.Mount(rootfs); err != nil {
+			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
+		}
+	}
+	process, err := newInit(
+		ctx,
+		s.config.Path,
+		s.config.WorkDir,
+		s.config.RuntimeRoot,
+		s.config.Namespace,
+		s.config.Criu,
+		s.config.SystemdCgroup,
+		s.platform,
+		config,
 	)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
+	if err := process.Create(ctx, config); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
 	// save the main task id and bundle to the shim for additional requests
 	s.id = r.ID
 	s.bundle = r.Bundle
@@ -414,9 +439,22 @@ func (s *Service) Checkpoint(ctx context.Context, r *shimapi.CheckpointTaskReque
 	if p == nil {
 		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
 	}
+	var options runctypes.CheckpointOptions
+	if r.Options != nil {
+		v, err := typeurl.UnmarshalAny(r.Options)
+		if err != nil {
+			return nil, err
+		}
+		options = *v.(*runctypes.CheckpointOptions)
+	}
 	if err := p.(*proc.Init).Checkpoint(ctx, &proc.CheckpointConfig{
 		Path:                     r.Path,
-		Options: r.Options,
+		Exit:                     options.Exit,
+		AllowOpenTCP:             options.OpenTcp,
+		AllowExternalUnixSockets: options.ExternalUnixSockets,
+		AllowTerminal:            options.Terminal,
+		FileLocks:                options.FileLocks,
+		EmptyNamespaces:          options.EmptyNamespaces,
 	}); err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
@@ -545,3 +583,32 @@ func getTopic(ctx context.Context, e interface{}) string {
 	}
 	return runtime.TaskUnknownTopic
 }
+
+func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu string, systemdCgroup bool, platform rproc.Platform, r *proc.CreateConfig) (*proc.Init, error) {
+	var options runctypes.CreateOptions
+	if r.Options != nil {
+		v, err := typeurl.UnmarshalAny(r.Options)
+		if err != nil {
+			return nil, err
+		}
+		options = *v.(*runctypes.CreateOptions)
+	}
+
+	rootfs := filepath.Join(path, "rootfs")
+	runtime := proc.NewRunc(runtimeRoot, path, namespace, r.Runtime, criu, systemdCgroup)
+	p := proc.New(r.ID, runtime, rproc.Stdio{
+		Stdin:    r.Stdin,
+		Stdout:   r.Stdout,
+		Stderr:   r.Stderr,
+		Terminal: r.Terminal,
+	})
+	p.Bundle = r.Bundle
+	p.Platform = platform
+	p.Rootfs = rootfs
+	p.WorkDir = workDir
+	p.IoUID = int(options.IoUid)
+	p.IoGID = int(options.IoGid)
+	p.NoPivotRoot = options.NoPivotRoot
+	p.NoNewKeyring = options.NoNewKeyring
+	return p, nil
+}

runtime/v1/shim/v1/shim.pb.go

@@ -1,11 +1,11 @@
 // Code generated by protoc-gen-gogo. DO NOT EDIT.
-// source: github.com/containerd/containerd/runtime/shim/v1/shim.proto
+// source: github.com/containerd/containerd/runtime/v1/shim/v1/shim.proto
 
 /*
 	Package shim is a generated protocol buffer package.
 
 	It is generated from these files:
-		github.com/containerd/containerd/runtime/shim/v1/shim.proto
+		github.com/containerd/containerd/runtime/v1/shim/v1/shim.proto
 
 	It has these top-level messages:
 		CreateTaskRequest
@@ -4352,80 +4352,80 @@ var (
 )
 
 func init() {
-	proto.RegisterFile("github.com/containerd/containerd/runtime/shim/v1/shim.proto", fileDescriptorShim)
+	proto.RegisterFile("github.com/containerd/containerd/runtime/v1/shim/v1/shim.proto", fileDescriptorShim)
 }
 
 var fileDescriptorShim = []byte{
-	// 1135 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4d, 0x6f, 0xdb, 0x46,
-	0x13, 0x36, 0x69, 0x7d, 0x8e, 0x22, 0xbf, 0xf6, 0xbe, 0x8e, 0xcb, 0x28, 0x80, 0x2c, 0x10, 0x68,
-	0xe0, 0xa2, 0x08, 0x55, 0xcb, 0x6d, 0xd2, 0x36, 0x40, 0x00, 0xdb, 0x09, 0x0a, 0xa3, 0x0d, 0x6c,
-	0xd0, 0x4e, 0x13, 0xb4, 0x28, 0x0c, 0x5a, 0x5c, 0x4b, 0x0b, 0x4b, 0x24, 0xc3, 0x5d, 0xba, 0x76,
-	0x4f, 0x3d, 0xf5, 0xdc, 0x9f, 0xd3, 0x9f, 0xe0, 0x43, 0x0e, 0x3d, 0xf6, 0x94, 0x36, 0xba, 0xf7,
-	0x3f, 0x14, 0xfb, 0x21, 0x93, 0x92, 0xcc, 0x90, 0xf2, 0xc5, 0xe2, 0xec, 0x3e, 0xb3, 0x3b, 0x3b,
-	0xcf, 0xb3, 0x33, 0x6b, 0x78, 0xd2, 0x23, 0xac, 0x1f, 0x9d, 0x58, 0x5d, 0x7f, 0xd8, 0xee, 0xfa,
-	0x1e, 0x73, 0x88, 0x87, 0x43, 0x37, 0xf9, 0x19, 0x46, 0x1e, 0x23, 0x43, 0xdc, 0xa6, 0x7d, 0x32,
-	0x6c, 0x9f, 0x6f, 0x8a, 0x5f, 0x2b, 0x08, 0x7d, 0xe6, 0xa3, 0x56, 0x0c, 0xb3, 0x14, 0xcc, 0x1a,
-	0x10, 0x2f, 0xba, 0xb0, 0x04, 0xe8, 0x7c, 0xb3, 0x71, 0xaf, 0xe7, 0xfb, 0xbd, 0x01, 0x6e, 0x0b,
-	0xfc, 0x49, 0x74, 0xda, 0x76, 0xbc, 0x4b, 0xe9, 0xdc, 0xb8, 0x3f, 0x3d, 0x85, 0x87, 0x01, 0x1b,
-	0x4f, 0xae, 0xf6, 0xfc, 0x9e, 0x2f, 0x3e, 0xdb, 0xfc, 0x4b, 0x8d, 0xae, 0x4f, 0xbb, 0xf0, 0x1d,
-	0x29, 0x73, 0x86, 0x81, 0x02, 0x3c, 0xca, 0x3c, 0x8d, 0x13, 0x90, 0x36, 0xbb, 0x0c, 0x30, 0x6d,
-	0x0f, 0xfd, 0xc8, 0x63, 0xca, 0xef, 0xeb, 0x39, 0xfc, 0x98, 0x43, 0xcf, 0xc4, 0x1f, 0xe9, 0x6b,
-	0xfe, 0xab, 0xc3, 0xca, 0x6e, 0x88, 0x1d, 0x86, 0x8f, 0x1c, 0x7a, 0x66, 0xe3, 0x37, 0x11, 0xa6,
-	0x0c, 0xad, 0x81, 0x4e, 0x5c, 0x43, 0x6b, 0x69, 0x1b, 0xd5, 0x9d, 0xd2, 0xe8, 0xdd, 0xba, 0xbe,
-	0xf7, 0xcc, 0xd6, 0x89, 0x8b, 0xd6, 0xa0, 0x74, 0x12, 0x79, 0xee, 0x00, 0x1b, 0x3a, 0x9f, 0xb3,
-	0x95, 0x85, 0x0c, 0x28, 0xab, 0x0c, 0x1a, 0x8b, 0x62, 0x62, 0x6c, 0xa2, 0x36, 0x94, 0x42, 0xdf,
-	0x67, 0xa7, 0xd4, 0x28, 0xb4, 0x16, 0x37, 0x6a, 0x9d, 0x8f, 0xac, 0x44, 0xd6, 0x45, 0x48, 0xd6,
-	0x0b, 0x7e, 0x14, 0x5b, 0xc1, 0x50, 0x03, 0x2a, 0x0c, 0x87, 0x43, 0xe2, 0x39, 0x03, 0xa3, 0xd8,
-	0xd2, 0x36, 0x2a, 0xf6, 0xb5, 0x8d, 0x56, 0xa1, 0x48, 0x99, 0x4b, 0x3c, 0xa3, 0x24, 0x36, 0x91,
-	0x06, 0x0f, 0x8a, 0x32, 0xd7, 0x8f, 0x98, 0x51, 0x96, 0x41, 0x49, 0x4b, 0x8d, 0xe3, 0x30, 0x34,
-	0x2a, 0xd7, 0xe3, 0x38, 0x0c, 0x51, 0x13, 0xa0, 0xdb, 0xc7, 0xdd, 0xb3, 0xc0, 0x27, 0x1e, 0x33,
-	0xaa, 0x62, 0x2e, 0x31, 0x82, 0x3e, 0x85, 0x95, 0xc0, 0x09, 0xb1, 0xc7, 0x8e, 0x13, 0x30, 0x10,
-	0xb0, 0x65, 0x39, 0xb1, 0x1b, 0x83, 0x2d, 0x28, 0xfb, 0x01, 0x23, 0xbe, 0x47, 0x8d, 0x5a, 0x4b,
-	0xdb, 0xa8, 0x75, 0x56, 0x2d, 0x49, 0xb3, 0x35, 0xa6, 0xd9, 0xda, 0xf6, 0x2e, 0xed, 0x31, 0xc8,
-	0x7c, 0x00, 0x28, 0x99, 0x6e, 0x1a, 0xf8, 0x1e, 0xc5, 0x68, 0x19, 0x16, 0x03, 0x95, 0xf0, 0xba,
-	0xcd, 0x3f, 0xcd, 0xdf, 0x34, 0x58, 0x7a, 0x86, 0x07, 0x98, 0xe1, 0x74, 0x10, 0x5a, 0x87, 0x1a,
-	0xbe, 0x20, 0xec, 0x98, 0x32, 0x87, 0x45, 0x54, 0x70, 0x52, 0xb7, 0x81, 0x0f, 0x1d, 0x8a, 0x11,
-	0xb4, 0x0d, 0x55, 0x6e, 0x61, 0xf7, 0xd8, 0x61, 0x82, 0x99, 0x5a, 0xa7, 0x31, 0x13, 0xdf, 0xd1,
-	0x58, 0x86, 0x3b, 0x95, 0xab, 0x77, 0xeb, 0x0b, 0xbf, 0xff, 0xbd, 0xae, 0xd9, 0x15, 0xe9, 0xb6,
-	0xcd, 0x4c, 0x0b, 0x56, 0x65, 0x1c, 0x07, 0xa1, 0xdf, 0xc5, 0x94, 0x66, 0x48, 0xc4, 0xfc, 0x43,
-	0x03, 0xf4, 0xfc, 0x02, 0x77, 0xf3, 0xc1, 0x27, 0xe8, 0xd6, 0xd3, 0xe8, 0x5e, 0xbc, 0x99, 0xee,
-	0x42, 0x0a, 0xdd, 0xc5, 0x09, 0xba, 0x37, 0xa0, 0x40, 0x03, 0xdc, 0x15, 0x9a, 0x49, 0xa3, 0x47,
-	0x20, 0xcc, 0xbb, 0xf0, 0xff, 0x89, 0xc8, 0x65, 0xde, 0xcd, 0xd7, 0xb0, 0x6c, 0x63, 0x4a, 0x7e,
-	0xc1, 0x07, 0xec, 0x32, 0xeb, 0x38, 0xab, 0x50, 0xfc, 0x99, 0xb8, 0xac, 0xaf, 0xb8, 0x90, 0x06,
-	0x0f, 0xad, 0x8f, 0x49, 0xaf, 0x2f, 0x39, 0xa8, 0xdb, 0xca, 0x32, 0x1f, 0xc0, 0x1d, 0x4e, 0x14,
-	0xce, 0xca, 0xe9, 0x5b, 0x1d, 0xea, 0x0a, 0xa8, 0xb4, 0x30, 0xef, 0x05, 0x55, 0xda, 0x59, 0x8c,
-	0xb5, 0xb3, 0xc5, 0xd3, 0x25, 0x64, 0xc3, 0xd3, 0xb8, 0xd4, 0xb9, 0x9f, 0xbc, 0x98, 0xe7, 0x9b,
-	0xea, 0x6e, 0x4a, 0x1d, 0xd9, 0x0a, 0x1a, 0x33, 0x52, 0xbc, 0x99, 0x91, 0x52, 0x0a, 0x23, 0xe5,
-	0x09, 0x46, 0x92, 0x9c, 0x57, 0xa6, 0x38, 0x9f, 0x92, 0x74, 0xf5, 0xc3, 0x92, 0x86, 0x5b, 0x49,
-	0x7a, 0x1f, 0x6a, 0xdf, 0x92, 0xc1, 0x20, 0x47, 0xb1, 0xa3, 0xa4, 0x37, 0x16, 0x66, 0xdd, 0x56,
-	0x16, 0xcf, 0xa5, 0x33, 0x18, 0x88, 0x5c, 0x56, 0x6c, 0xfe, 0x69, 0x3e, 0x85, 0xa5, 0xdd, 0x81,
-	0x4f, 0xf1, 0xde, 0x7e, 0x0e, 0x7d, 0xc8, 0x04, 0x4a, 0xad, 0x4b, 0xc3, 0xfc, 0x04, 0xfe, 0xf7,
-	0x1d, 0xa1, 0xec, 0x80, 0xb8, 0x99, 0xd7, 0xcb, 0x86, 0xe5, 0x18, 0xaa, 0xc4, 0xf0, 0x14, 0xaa,
-	0x81, 0xd4, 0x2c, 0xa6, 0x86, 0x26, 0xca, 0x6c, 0xeb, 0x46, 0x36, 0x95, 0xb2, 0xf7, 0xbc, 0x53,
-	0xdf, 0x8e, 0x5d, 0xcc, 0x1f, 0xe1, 0x6e, 0x5c, 0xd1, 0x92, 0x6d, 0x00, 0x41, 0x21, 0x70, 0x58,
-	0x5f, 0x86, 0x61, 0x8b, 0xef, 0x64, 0xc1, 0xd3, 0xf3, 0x14, 0xbc, 0x87, 0xb0, 0x7c, 0xd8, 0x27,
-	0x43, 0xb1, 0xe7, 0x38, 0xe0, 0x7b, 0x50, 0xe1, 0x2d, 0xf6, 0x38, 0x2e, 0x67, 0x65, 0x6e, 0x1f,
-	0x10, 0xd7, 0xfc, 0x06, 0x56, 0x5e, 0x06, 0xee, 0x54, 0x3b, 0xea, 0x40, 0x35, 0xc4, 0xd4, 0x8f,
-	0xc2, 0xae, 0x38, 0x60, 0xfa, 0xae, 0x31, 0x4c, 0xdd, 0xad, 0x90, 0x65, 0x25, 0xf4, 0x2b, 0x71,
-	0xb5, 0x38, 0x2e, 0xe3, 0x6a, 0xa9, 0x2b, 0xa4, 0xc7, 0x35, 0xfa, 0x63, 0xa8, 0xbd, 0x72, 0x48,
-	0xe6, 0x0e, 0x21, 0xdc, 0x91, 0x30, 0xb5, 0xc1, 0x94, 0xc4, 0xb5, 0x0f, 0x4b, 0x5c, 0xbf, 0x8d,
-	0xc4, 0x3b, 0x6f, 0x6b, 0x50, 0xe0, 0x69, 0x47, 0x7d, 0x28, 0x8a, 0xca, 0x81, 0x2c, 0x2b, 0xeb,
-	0xb9, 0x63, 0x25, 0x6b, 0x51, 0xa3, 0x9d, 0x1b, 0xaf, 0x8e, 0x45, 0xa1, 0x24, 0x3b, 0x1b, 0xda,
-	0xca, 0x76, 0x9d, 0x79, 0x72, 0x34, 0x3e, 0x9f, 0xcf, 0x49, 0x6d, 0x2a, 0x8f, 0x17, 0xb2, 0x9c,
-	0xc7, 0xbb, 0x96, 0x43, 0xce, 0xe3, 0x25, 0x64, 0x61, 0x43, 0x49, 0xf6, 0x41, 0xb4, 0x36, 0xc3,
-	0xc5, 0x73, 0xfe, 0xf6, 0x6b, 0x7c, 0x96, 0xbd, 0xe4, 0x54, 0x47, 0xbf, 0x84, 0xfa, 0x44, 0x6f,
-	0x45, 0x8f, 0xf2, 0x2e, 0x31, 0xd9, 0x5d, 0x6f, 0xb1, 0xf5, 0x1b, 0xa8, 0x8c, 0xeb, 0x08, 0xda,
-	0xcc, 0xf6, 0x9e, 0x2a, 0x4f, 0x8d, 0xce, 0x3c, 0x2e, 0x6a, 0xcb, 0xc7, 0x50, 0x3c, 0x70, 0x22,
-	0x9a, 0x9e, 0xc0, 0x94, 0x71, 0xf4, 0x25, 0x94, 0x6c, 0x4c, 0xa3, 0xe1, 0xfc, 0x9e, 0x3f, 0x01,
-	0x24, 0xde, 0x6a, 0x8f, 0x73, 0x48, 0xec, 0xa6, 0x3a, 0x98, 0xba, 0xfc, 0x0b, 0x28, 0xf0, 0x46,
-	0x82, 0x1e, 0x66, 0x2f, 0x9c, 0x68, 0x38, 0xa9, 0xcb, 0x1d, 0x41, 0x81, 0xbf, 0x3f, 0x50, 0x8e,
-	0xab, 0x30, 0xfb, 0xc2, 0x4a, 0x5d, 0xf5, 0x15, 0x54, 0xaf, 0x9f, 0x2f, 0x28, 0x07, 0x6f, 0xd3,
-	0x6f, 0x9d, 0xd4, 0x85, 0x0f, 0xa1, 0xac, 0xba, 0x1e, 0xca, 0xa1, 0xbf, 0xc9, 0x06, 0x99, 0xba,
-	0xe8, 0xf7, 0x50, 0x19, 0xb7, 0x8b, 0x54, 0xb6, 0x73, 0x1c, 0x62, 0xa6, 0xe5, 0xbc, 0x84, 0x92,
-	0xec, 0x2b, 0x79, 0xaa, 0xd3, 0x4c, 0x07, 0x4a, 0x0d, 0x17, 0x43, 0x81, 0xd7, 0xf6, 0x3c, 0x0a,
-	0x48, 0xb4, 0x8a, 0x86, 0x95, 0x17, 0x2e, 0xa3, 0xdf, 0xd9, 0xbf, 0x7a, 0xdf, 0x5c, 0xf8, 0xeb,
-	0x7d, 0x73, 0xe1, 0xd7, 0x51, 0x53, 0xbb, 0x1a, 0x35, 0xb5, 0x3f, 0x47, 0x4d, 0xed, 0x9f, 0x51,
-	0x53, 0xfb, 0xe1, 0x8b, 0x79, 0xff, 0x03, 0x7e, 0xc2, 0x7f, 0x5f, 0xeb, 0x27, 0x25, 0x71, 0x92,
-	0xad, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x5b, 0x88, 0x3a, 0x56, 0x43, 0x0f, 0x00, 0x00,
+	// 1133 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x57, 0x4f, 0x4f, 0x1b, 0x47,
+	0x14, 0x67, 0x17, 0xff, 0x7d, 0x8e, 0x29, 0x4c, 0x09, 0xdd, 0x38, 0x92, 0xb1, 0x56, 0x6a, 0x44,
+	0x55, 0x65, 0x5d, 0x4c, 0x95, 0xa4, 0xad, 0x84, 0x04, 0x24, 0xaa, 0x50, 0x1b, 0x05, 0x2d, 0xa4,
+	0x89, 0x5a, 0x55, 0x68, 0xf1, 0x0e, 0xf6, 0x08, 0x7b, 0x67, 0xb3, 0x33, 0x4b, 0xa1, 0xa7, 0x9e,
+	0x7a, 0xee, 0xc7, 0xe9, 0x47, 0xe0, 0x90, 0x43, 0x8f, 0x3d, 0xa5, 0x0d, 0xf7, 0x7e, 0x87, 0x6a,
+	0xfe, 0x18, 0xaf, 0x6d, 0x36, 0xbb, 0x70, 0xc1, 0xfb, 0x66, 0x7e, 0x6f, 0xe6, 0xcd, 0xfb, 0xfd,
+	0xe6, 0xbd, 0x01, 0x36, 0x7b, 0x84, 0xf7, 0xe3, 0x23, 0xa7, 0x4b, 0x87, 0xed, 0x2e, 0x0d, 0xb8,
+	0x47, 0x02, 0x1c, 0xf9, 0xc9, 0xcf, 0x28, 0x0e, 0x38, 0x19, 0xe2, 0xf6, 0xe9, 0x7a, 0x9b, 0xf5,
+	0xc9, 0x70, 0xf4, 0xeb, 0x84, 0x11, 0xe5, 0x14, 0xb5, 0xc6, 0x48, 0x47, 0x23, 0x9d, 0x01, 0x09,
+	0xe2, 0x33, 0x47, 0x82, 0x4e, 0xd7, 0x1b, 0xf7, 0x7a, 0x94, 0xf6, 0x06, 0xb8, 0x2d, 0xf1, 0x47,
+	0xf1, 0x71, 0xdb, 0x0b, 0xce, 0x95, 0x73, 0xe3, 0xfe, 0xf4, 0x14, 0x1e, 0x86, 0x7c, 0x34, 0xb9,
+	0xdc, 0xa3, 0x3d, 0x2a, 0x3f, 0xdb, 0xe2, 0x4b, 0x8f, 0xae, 0x4e, 0xbb, 0x88, 0x1d, 0x19, 0xf7,
+	0x86, 0xa1, 0x06, 0x3c, 0xca, 0x3c, 0x90, 0x17, 0x92, 0x36, 0x3f, 0x0f, 0x31, 0x6b, 0x0f, 0x69,
+	0x1c, 0x70, 0xed, 0xf7, 0xf5, 0x0d, 0xfc, 0xb8, 0xc7, 0x4e, 0xe4, 0x1f, 0xe5, 0x6b, 0xff, 0x67,
+	0xc2, 0xd2, 0x4e, 0x84, 0x3d, 0x8e, 0x0f, 0x3c, 0x76, 0xe2, 0xe2, 0x37, 0x31, 0x66, 0x1c, 0xad,
+	0x80, 0x49, 0x7c, 0xcb, 0x68, 0x19, 0x6b, 0xd5, 0xed, 0xd2, 0xe5, 0xbb, 0x55, 0x73, 0xf7, 0xa9,
+	0x6b, 0x12, 0x1f, 0xad, 0x40, 0xe9, 0x28, 0x0e, 0xfc, 0x01, 0xb6, 0x4c, 0x31, 0xe7, 0x6a, 0x0b,
+	0x59, 0x50, 0xd6, 0x19, 0xb4, 0xe6, 0xe5, 0xc4, 0xc8, 0x44, 0x6d, 0x28, 0x45, 0x94, 0xf2, 0x63,
+	0x66, 0x15, 0x5a, 0xf3, 0x6b, 0xb5, 0xce, 0x27, 0x4e, 0x22, 0xeb, 0x32, 0x24, 0xe7, 0xb9, 0x38,
+	0x8a, 0xab, 0x61, 0xa8, 0x01, 0x15, 0x8e, 0xa3, 0x21, 0x09, 0xbc, 0x81, 0x55, 0x6c, 0x19, 0x6b,
+	0x15, 0xf7, 0xca, 0x46, 0xcb, 0x50, 0x64, 0xdc, 0x27, 0x81, 0x55, 0x92, 0x9b, 0x28, 0x43, 0x04,
+	0xc5, 0xb8, 0x4f, 0x63, 0x6e, 0x95, 0x55, 0x50, 0xca, 0xd2, 0xe3, 0x38, 0x8a, 0xac, 0xca, 0xd5,
+	0x38, 0x8e, 0x22, 0xd4, 0x04, 0xe8, 0xf6, 0x71, 0xf7, 0x24, 0xa4, 0x24, 0xe0, 0x56, 0x55, 0xce,
+	0x25, 0x46, 0xd0, 0xe7, 0xb0, 0x14, 0x7a, 0x11, 0x0e, 0xf8, 0x61, 0x02, 0x06, 0x12, 0xb6, 0xa8,
+	0x26, 0x76, 0xc6, 0x60, 0x07, 0xca, 0x34, 0xe4, 0x84, 0x06, 0xcc, 0xaa, 0xb5, 0x8c, 0xb5, 0x5a,
+	0x67, 0xd9, 0x51, 0x34, 0x3b, 0x23, 0x9a, 0x9d, 0xad, 0xe0, 0xdc, 0x1d, 0x81, 0xec, 0x07, 0x80,
+	0x92, 0xe9, 0x66, 0x21, 0x0d, 0x18, 0x46, 0x8b, 0x30, 0x1f, 0xea, 0x84, 0xd7, 0x5d, 0xf1, 0x69,
+	0xff, 0x6e, 0xc0, 0xc2, 0x53, 0x3c, 0xc0, 0x1c, 0xa7, 0x83, 0xd0, 0x2a, 0xd4, 0xf0, 0x19, 0xe1,
+	0x87, 0x8c, 0x7b, 0x3c, 0x66, 0x92, 0x93, 0xba, 0x0b, 0x62, 0x68, 0x5f, 0x8e, 0xa0, 0x2d, 0xa8,
+	0x0a, 0x0b, 0xfb, 0x87, 0x1e, 0x97, 0xcc, 0xd4, 0x3a, 0x8d, 0x99, 0xf8, 0x0e, 0x46, 0x32, 0xdc,
+	0xae, 0x5c, 0xbc, 0x5b, 0x9d, 0xfb, 0xe3, 0x9f, 0x55, 0xc3, 0xad, 0x28, 0xb7, 0x2d, 0x6e, 0x3b,
+	0xb0, 0xac, 0xe2, 0xd8, 0x8b, 0x68, 0x17, 0x33, 0x96, 0x21, 0x11, 0xfb, 0x4f, 0x03, 0xd0, 0xb3,
+	0x33, 0xdc, 0xcd, 0x07, 0x9f, 0xa0, 0xdb, 0x4c, 0xa3, 0x7b, 0xfe, 0x7a, 0xba, 0x0b, 0x29, 0x74,
+	0x17, 0x27, 0xe8, 0x5e, 0x83, 0x02, 0x0b, 0x71, 0x57, 0x6a, 0x26, 0x8d, 0x1e, 0x89, 0xb0, 0xef,
+	0xc2, 0xc7, 0x13, 0x91, 0xab, 0xbc, 0xdb, 0xaf, 0x61, 0xd1, 0xc5, 0x8c, 0xfc, 0x8a, 0xf7, 0xf8,
+	0x79, 0xd6, 0x71, 0x96, 0xa1, 0xf8, 0x0b, 0xf1, 0x79, 0x5f, 0x73, 0xa1, 0x0c, 0x11, 0x5a, 0x1f,
+	0x93, 0x5e, 0x5f, 0x71, 0x50, 0x77, 0xb5, 0x65, 0x3f, 0x80, 0x3b, 0x82, 0x28, 0x9c, 0x95, 0xd3,
+	0xb7, 0x26, 0xd4, 0x35, 0x50, 0x6b, 0xe1, 0xa6, 0x17, 0x54, 0x6b, 0x67, 0x7e, 0xac, 0x9d, 0x0d,
+	0x91, 0x2e, 0x29, 0x1b, 0x91, 0xc6, 0x85, 0xce, 0xfd, 0xe4, 0xc5, 0x3c, 0x5d, 0xd7, 0x77, 0x53,
+	0xe9, 0xc8, 0xd5, 0xd0, 0x31, 0x23, 0xc5, 0xeb, 0x19, 0x29, 0xa5, 0x30, 0x52, 0x9e, 0x60, 0x24,
+	0xc9, 0x79, 0x65, 0x8a, 0xf3, 0x29, 0x49, 0x57, 0x3f, 0x2c, 0x69, 0xb8, 0x95, 0xa4, 0x5f, 0x40,
+	0xed, 0x3b, 0x32, 0x18, 0xe4, 0x28, 0x76, 0x8c, 0xf4, 0x46, 0xc2, 0xac, 0xbb, 0xda, 0x12, 0xb9,
+	0xf4, 0x06, 0x03, 0x99, 0xcb, 0x8a, 0x2b, 0x3e, 0xed, 0x4d, 0x58, 0xd8, 0x19, 0x50, 0x86, 0x77,
+	0x5f, 0xe4, 0xd0, 0x87, 0x4a, 0xa0, 0xd2, 0xba, 0x32, 0xec, 0xcf, 0xe0, 0xa3, 0xef, 0x09, 0xe3,
+	0x7b, 0xc4, 0xcf, 0xbc, 0x5e, 0x2e, 0x2c, 0x8e, 0xa1, 0x5a, 0x0c, 0x9b, 0x50, 0x0d, 0x95, 0x66,
+	0x31, 0xb3, 0x0c, 0x59, 0x66, 0x5b, 0xd7, 0xb2, 0xa9, 0x95, 0xbd, 0x1b, 0x1c, 0x53, 0x77, 0xec,
+	0x62, 0xff, 0x04, 0x77, 0xc7, 0x15, 0x2d, 0xd9, 0x06, 0x10, 0x14, 0x42, 0x8f, 0xf7, 0x55, 0x18,
+	0xae, 0xfc, 0x4e, 0x16, 0x3c, 0x33, 0x4f, 0xc1, 0x7b, 0x08, 0x8b, 0xfb, 0x7d, 0x32, 0x94, 0x7b,
+	0x8e, 0x02, 0xbe, 0x07, 0x15, 0xd1, 0x62, 0x0f, 0xc7, 0xe5, 0xac, 0x2c, 0xec, 0x3d, 0xe2, 0xdb,
+	0xdf, 0xc2, 0xd2, 0xcb, 0xd0, 0x9f, 0x6a, 0x47, 0x1d, 0xa8, 0x46, 0x98, 0xd1, 0x38, 0xea, 0xca,
+	0x03, 0xa6, 0xef, 0x3a, 0x86, 0xe9, 0xbb, 0x15, 0xf1, 0xac, 0x84, 0x7e, 0x25, 0xaf, 0x96, 0xc0,
+	0x65, 0x5c, 0x2d, 0x7d, 0x85, 0xcc, 0x71, 0x8d, 0xfe, 0x14, 0x6a, 0xaf, 0x3c, 0x92, 0xb9, 0x43,
+	0x04, 0x77, 0x14, 0x4c, 0x6f, 0x30, 0x25, 0x71, 0xe3, 0xc3, 0x12, 0x37, 0x6f, 0x23, 0xf1, 0xce,
+	0xdb, 0x1a, 0x14, 0x44, 0xda, 0x51, 0x1f, 0x8a, 0xb2, 0x72, 0x20, 0xc7, 0xc9, 0x7a, 0xee, 0x38,
+	0xc9, 0x5a, 0xd4, 0x68, 0xe7, 0xc6, 0xeb, 0x63, 0x31, 0x28, 0xa9, 0xce, 0x86, 0x36, 0xb2, 0x5d,
+	0x67, 0x9e, 0x1c, 0x8d, 0x2f, 0x6f, 0xe6, 0xa4, 0x37, 0x55, 0xc7, 0x8b, 0x78, 0xce, 0xe3, 0x5d,
+	0xc9, 0x21, 0xe7, 0xf1, 0x12, 0xb2, 0x70, 0xa1, 0xa4, 0xfa, 0x20, 0x5a, 0x99, 0xe1, 0xe2, 0x99,
+	0x78, 0xfb, 0x35, 0xbe, 0xc8, 0x5e, 0x72, 0xaa, 0xa3, 0x9f, 0x43, 0x7d, 0xa2, 0xb7, 0xa2, 0x47,
+	0x79, 0x97, 0x98, 0xec, 0xae, 0xb7, 0xd8, 0xfa, 0x0d, 0x54, 0x46, 0x75, 0x04, 0xad, 0x67, 0x7b,
+	0x4f, 0x95, 0xa7, 0x46, 0xe7, 0x26, 0x2e, 0x7a, 0xcb, 0xc7, 0x50, 0xdc, 0xf3, 0x62, 0x96, 0x9e,
+	0xc0, 0x94, 0x71, 0xf4, 0x04, 0x4a, 0x2e, 0x66, 0xf1, 0xf0, 0xe6, 0x9e, 0x3f, 0x03, 0x24, 0xde,
+	0x6a, 0x8f, 0x73, 0x48, 0xec, 0xba, 0x3a, 0x98, 0xba, 0xfc, 0x73, 0x28, 0x88, 0x46, 0x82, 0x1e,
+	0x66, 0x2f, 0x9c, 0x68, 0x38, 0xa9, 0xcb, 0x1d, 0x40, 0x41, 0xbc, 0x3f, 0x50, 0x8e, 0xab, 0x30,
+	0xfb, 0xc2, 0x4a, 0x5d, 0xf5, 0x15, 0x54, 0xaf, 0x9e, 0x2f, 0x28, 0x07, 0x6f, 0xd3, 0x6f, 0x9d,
+	0xd4, 0x85, 0xf7, 0xa1, 0xac, 0xbb, 0x1e, 0xca, 0xa1, 0xbf, 0xc9, 0x06, 0x99, 0xba, 0xe8, 0x0f,
+	0x50, 0x19, 0xb5, 0x8b, 0x54, 0xb6, 0x73, 0x1c, 0x62, 0xa6, 0xe5, 0xbc, 0x84, 0x92, 0xea, 0x2b,
+	0x79, 0xaa, 0xd3, 0x4c, 0x07, 0x4a, 0x0d, 0x17, 0x43, 0x41, 0xd4, 0xf6, 0x3c, 0x0a, 0x48, 0xb4,
+	0x8a, 0x86, 0x93, 0x17, 0xae, 0xa2, 0xdf, 0x76, 0x2f, 0xde, 0x37, 0xe7, 0xfe, 0x7e, 0xdf, 0x9c,
+	0xfb, 0xed, 0xb2, 0x69, 0x5c, 0x5c, 0x36, 0x8d, 0xbf, 0x2e, 0x9b, 0xc6, 0xbf, 0x97, 0x4d, 0xe3,
+	0xc7, 0x27, 0xb7, 0xf8, 0x27, 0xf8, 0x1b, 0xf1, 0xfb, 0xda, 0x3c, 0x2a, 0xc9, 0xc3, 0x6c, 0xfc,
+	0x1f, 0x00, 0x00, 0xff, 0xff, 0x64, 0x52, 0x86, 0xc0, 0x49, 0x0f, 0x00, 0x00,
 }

runtime/v1/shim/v1/shim.proto

@@ -9,7 +9,7 @@ import "google/protobuf/timestamp.proto";
 import "github.com/containerd/containerd/api/types/mount.proto";
 import "github.com/containerd/containerd/api/types/task/task.proto";
 
-option go_package = "github.com/containerd/containerd/runtime/shim/v1;shim";
+option go_package = "github.com/containerd/containerd/runtime/v1/shim/v1;shim";
 
 // Shim service is launched for each container and is responsible for owning the IO
 // for the container and its additional processes.  The shim is also the parent of

runtime/v2/README.md

@@ -0,0 +1,166 @@
+# Runtime v2
+
+Runtime v2 introduces a first class shim API for runtime authors to integrate with containerd.
+The shim API is minimal and scoped to the execution lifecycle of a container.
+
+## Binary Naming
+
+Users specify the runtime they wish to use when creating a container.
+The runtime can also be changed via a container update.
+
+```bash
+> ctr run --runtime io.containerd.runc.v1
+```
+
+When a user specifies a runtime name, `io.containerd.runc.v1`, they will specify the name and version of the runtime.
+This will be trasnlated by containerd into a binary name for the shim.
+
+`io.containerd.runc.v1` -> `containerd-shim-runc-v1`
+
+containerd keeps the `containerd-shim-*` prefix so that users can `ps aux | grep containerd-shim` to see running shims on their system.
+
+## Shim Authoring
+
+This section is dedicated to runtime authors wishing to build a shim.
+It will detail how the API works and different considerations when building shim.
+
+### Commands
+
+Container information is provided to a shim in two ways.
+The OCI Runtime Bundle and on the `Create` rpc request.
+
+#### `start`
+
+Each shim MUST implement a `start` subcommand.
+This command will launch new shims.
+The start command MUST accept the following flags:
+
+* `-namespace` the namespace for the container
+* `-id` the id of the container
+* `-address` the address of the containerd's main socket
+* `-publish-binary` the binary path to publish events back to containerd
+
+The start command, as well as all binary calls to the shim, has the bundle for the container set as the `cwd`.
+
+The start command MUST return an address to a shim for containerd to issue API requests for container operations.
+
+The start command can either start a new shim or return an address to an existing shim based on the shim's logic.
+
+#### `delete`
+
+Each shim MUST implement a `delete` subcommand.
+This command allows containerd to delete any container resources created, mounted, and/or run by a shim when containerd can no longer communicate over rpc.
+This happens if a shim is SIGKILL'd with a running container.
+These resources will need to be cleaned up when containerd looses the connection to a shim.
+This is also used when containerd boots and reconnects to shims.
+If a bundle is still on disk but containerd cannot connect to a shim, the delete command is invoked.
+
+The delete command MUST accept the following flags:
+
+* `-namespace` the namespace for the container
+* `-id` the id of the container
+* `-address` the address of the containerd's main socket
+* `-publish-binary` the binary path to publish events back to containerd
+
+The delete command will be executed in the container's bundle as its `cwd`.
+
+### Host Level Shim Configuration
+
+containerd does not provide any host level configuration for shims via the API.
+If a shim needs configuration from the user with host level information across all instances, a shim specific configuration file can be setup.
+
+### Container Level Shim Configuration
+
+On the create request, there is a generic `*protobuf.Any` that allows a user to specify container level configuration for the shim.
+
+```proto
+message CreateTaskRequest {
+	string id = 1;
+	...
+	google.protobuf.Any options = 10;
+}
+```
+
+A shim author can create their own protobuf message for configuration and clients can import and provide this information is needed.
+
+### I/O
+
+I/O for a container is provided by the client to the shim via fifo on Linux, named pipes on Windows, or log files on disk.
+The paths to these files are provided on the `Create` rpc for the initial creation and on the `Exec` rpc for additional processes.
+
+```proto
+message CreateTaskRequest {
+	string id = 1;
+	bool terminal = 4;
+	string stdin = 5;
+	string stdout = 6;
+	string stderr = 7;
+}
+```
+
+```proto
+message ExecProcessRequest {
+	string id = 1;
+	string exec_id = 2;
+	bool terminal = 3;
+	string stdin = 4;
+	string stdout = 5;
+	string stderr = 6;
+}
+```
+
+Containers that are to be launched with an interactive terminal will have the `terminal` field set to `true`, data is still copied over the files(fifos,pipes) in the same way as non interactive containers.
+
+### Root Filesystems
+
+The root filesytems for the containers is provided by on the `Create` rpc.
+Shims are responsible for managing the lifecycle of the filesystem mount during the lifecycle of a container.
+
+```proto
+message CreateTaskRequest {
+	string id = 1;
+	string bundle = 2;
+	repeated containerd.types.Mount rootfs = 3;
+	...
+}
+```
+
+The mount protobuf message is:
+
+```proto
+message Mount {
+	// Type defines the nature of the mount.
+	string type = 1;
+	// Source specifies the name of the mount. Depending on mount type, this
+	// may be a volume name or a host path, or even ignored.
+	string source = 2;
+	// Target path in container
+	string target = 3;
+	// Options specifies zero or more fstab style mount options.
+	repeated string options = 4;
+}
+```
+
+Shims are responsible for mounting the filesystem into the `rootfs/` directory of the bundle.
+Shims are also responsible for unmounting of the filesystem.
+During a `delete` binary call, the shim MUST ensure that filesystem is also unmounted.
+Filesystems are provided by the containerd snapshotters.
+
+### Events
+
+The shim MUST publish a `runtime.TaskExitEventTopic` when the container exits.
+If the shim collects Out of Memory events, it SHOULD also publish a `runtime.TaskOOMEventTopic`.
+
+### Other
+
+#### Unsupported rpcs
+
+If a shim does not or cannot implement an rpc call, it MUST return a `github.com/containerd/containerd/errdefs.ErrNotImplemented` error.
+
+#### ttrpc
+
+[ttrpc](https://github.com/containerd/ttrpc) is the only currently supported protocol for shims.
+It works with standard protobufs and GRPC services as well as generating clients.
+The only difference between grpc and ttrpc is the wire protocol.
+ttrpc removes the http stack in order to save memory and binary size to keep shims small.
+It is recommended to use ttrpc in your shim but grpc support is also in development.

runtime/v2/binary.go

@@ -0,0 +1,106 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"context"
+	"strings"
+
+	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/events/exchange"
+	"github.com/containerd/containerd/runtime"
+	client "github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	"github.com/pkg/errors"
+)
+
+func shimBinary(ctx context.Context, bundle *Bundle, runtime, containerdAddress string, events *exchange.Exchange, rt *runtime.TaskList) *binary {
+	return &binary{
+		bundle:            bundle,
+		runtime:           runtime,
+		containerdAddress: containerdAddress,
+		events:            events,
+		rtTasks:           rt,
+	}
+}
+
+type binary struct {
+	runtime           string
+	containerdAddress string
+	bundle            *Bundle
+	events            *exchange.Exchange
+	rtTasks           *runtime.TaskList
+}
+
+func (b *binary) Start(ctx context.Context) (*shim, error) {
+	cmd, err := client.Command(ctx, b.runtime, b.containerdAddress, b.bundle.Path, "-id", b.bundle.ID, "start")
+	if err != nil {
+		return nil, err
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrapf(err, "%s", out)
+	}
+	address := strings.TrimSpace(string(out))
+	conn, err := client.Connect(address, client.AnonDialer)
+	if err != nil {
+		return nil, err
+	}
+	client := ttrpc.NewClient(conn)
+	client.OnClose(func() { conn.Close() })
+	return &shim{
+		bundle:  b.bundle,
+		client:  client,
+		task:    task.NewTaskClient(client),
+		events:  b.events,
+		rtTasks: b.rtTasks,
+	}, nil
+}
+
+func (b *binary) Delete(ctx context.Context) (*runtime.Exit, error) {
+	cmd, err := client.Command(ctx, b.runtime, b.containerdAddress, b.bundle.Path, "-id", b.bundle.ID, "delete")
+	if err != nil {
+		return nil, err
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrapf(err, "%s", out)
+	}
+	var response task.DeleteResponse
+	if err := response.Unmarshal(out); err != nil {
+		return nil, err
+	}
+	if err := b.bundle.Delete(); err != nil {
+		return nil, err
+	}
+	// remove self from the runtime task list
+	// this seems dirty but it cleans up the API across runtimes, tasks, and the service
+	b.rtTasks.Delete(ctx, b.bundle.ID)
+	// shim will send the exit event
+	b.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
+		ContainerID: b.bundle.ID,
+		ExitStatus:  response.ExitStatus,
+		ExitedAt:    response.ExitedAt,
+		Pid:         response.Pid,
+	})
+	return &runtime.Exit{
+		Status:    response.ExitStatus,
+		Timestamp: response.ExitedAt,
+		Pid:       response.Pid,
+	}, nil
+}

runtime/v2/bundle.go

@@ -0,0 +1,120 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd/identifiers"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/pkg/errors"
+)
+
+const configFilename = "config.json"
+
+// LoadBundle loads an existing bundle from disk
+func LoadBundle(ctx context.Context, root, id string) (*Bundle, error) {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return &Bundle{
+		ID:        id,
+		Path:      filepath.Join(root, ns, id),
+		Namespace: ns,
+	}, nil
+}
+
+// NewBundle returns a new bundle on disk
+func NewBundle(ctx context.Context, root, state, id string, spec []byte) (b *Bundle, err error) {
+	if err := identifiers.Validate(id); err != nil {
+		return nil, errors.Wrapf(err, "invalid task id %s", id)
+	}
+
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	work := filepath.Join(root, ns, id)
+	b = &Bundle{
+		ID:        id,
+		Path:      filepath.Join(state, ns, id),
+		Namespace: ns,
+	}
+	paths := []string{b.Path, work}
+	// create base directories
+	for _, d := range paths {
+		if err := os.MkdirAll(filepath.Dir(d), 0711); err != nil {
+			return nil, err
+		}
+		if err := os.Mkdir(d, 0711); err != nil {
+			return nil, err
+		}
+	}
+	defer func() {
+		if err != nil {
+			for _, d := range paths {
+				os.RemoveAll(d)
+			}
+		}
+	}()
+	// create rootfs dir
+	if err := os.Mkdir(filepath.Join(b.Path, "rootfs"), 0711); err != nil {
+		return nil, err
+	}
+	// symlink workdir
+	if err := os.Symlink(work, filepath.Join(b.Path, "work")); err != nil {
+		return nil, err
+	}
+	// write the spec to the bundle
+	err = ioutil.WriteFile(filepath.Join(b.Path, configFilename), spec, 0666)
+	return b, err
+}
+
+// Bundle represents an OCI bundle
+type Bundle struct {
+	// ID of the bundle
+	ID string
+	// Path to the bundle
+	Path string
+	// Namespace of the bundle
+	Namespace string
+}
+
+// Delete a bundle atomically
+func (b *Bundle) Delete() error {
+	work, werr := os.Readlink(filepath.Join(b.Path, "work"))
+	err := os.RemoveAll(b.Path)
+	if err == nil {
+		if werr == nil {
+			return os.RemoveAll(work)
+		}
+		return nil
+	}
+	// error removing the bundle path; still attempt removing work dir
+	var err2 error
+	if werr == nil {
+		err2 = os.RemoveAll(work)
+		if err2 == nil {
+			return err
+		}
+	}
+	return errors.Wrapf(err, "failed to remove both bundle and workdir locations: %v", err2)
+}

runtime/v2/manager.go

@@ -0,0 +1,211 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/boltdb/bolt"
+	"github.com/containerd/containerd/containers"
+	"github.com/containerd/containerd/events/exchange"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/metadata"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/containerd/plugin"
+	"github.com/containerd/containerd/runtime"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+func init() {
+	plugin.Register(&plugin.Registration{
+		Type: plugin.RuntimePluginV2,
+		ID:   "task",
+		Requires: []plugin.Type{
+			plugin.MetadataPlugin,
+		},
+		InitFn: func(ic *plugin.InitContext) (interface{}, error) {
+			ic.Meta.Platforms = []ocispec.Platform{platforms.DefaultSpec()}
+			if err := os.MkdirAll(ic.Root, 0711); err != nil {
+				return nil, err
+			}
+			if err := os.MkdirAll(ic.State, 0711); err != nil {
+				return nil, err
+			}
+			m, err := ic.Get(plugin.MetadataPlugin)
+			if err != nil {
+				return nil, err
+			}
+			return New(ic.Context, ic.Root, ic.State, ic.Address, ic.Events, m.(*metadata.DB))
+		},
+	})
+}
+
+// New task manager for v2 shims
+func New(ctx context.Context, root, state, containerdAddress string, events *exchange.Exchange, db *metadata.DB) (*TaskManager, error) {
+	for _, d := range []string{root, state} {
+		if err := os.MkdirAll(d, 0711); err != nil {
+			return nil, err
+		}
+	}
+	m := &TaskManager{
+		root:              root,
+		state:             state,
+		containerdAddress: containerdAddress,
+		tasks:             runtime.NewTaskList(),
+		events:            events,
+		db:                db,
+	}
+	if err := m.loadExistingTasks(ctx); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// TaskManager manages v2 shim's and their tasks
+type TaskManager struct {
+	root              string
+	state             string
+	containerdAddress string
+
+	tasks  *runtime.TaskList
+	events *exchange.Exchange
+	db     *metadata.DB
+}
+
+// ID of the task manager
+func (m *TaskManager) ID() string {
+	return fmt.Sprintf("%s.%s", plugin.RuntimePluginV2, "task")
+}
+
+// Create a new task
+func (m *TaskManager) Create(ctx context.Context, id string, opts runtime.CreateOpts) (_ runtime.Task, err error) {
+	bundle, err := NewBundle(ctx, m.root, m.state, id, opts.Spec.Value)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			bundle.Delete()
+		}
+	}()
+	b := shimBinary(ctx, bundle, opts.Runtime, m.containerdAddress, m.events, m.tasks)
+	shim, err := b.Start(ctx)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			shim.Shutdown(ctx)
+			shim.Close()
+		}
+	}()
+	t, err := shim.Create(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+	m.tasks.Add(ctx, t)
+	return t, nil
+}
+
+// Get a specific task
+func (m *TaskManager) Get(ctx context.Context, id string) (runtime.Task, error) {
+	return m.tasks.Get(ctx, id)
+}
+
+// Tasks lists all tasks
+func (m *TaskManager) Tasks(ctx context.Context, all bool) ([]runtime.Task, error) {
+	return m.tasks.GetAll(ctx, all)
+}
+
+func (m *TaskManager) loadExistingTasks(ctx context.Context) error {
+	nsDirs, err := ioutil.ReadDir(m.state)
+	if err != nil {
+		return err
+	}
+	for _, nsd := range nsDirs {
+		if !nsd.IsDir() {
+			continue
+		}
+		ns := nsd.Name()
+		log.G(ctx).WithField("namespace", ns).Debug("loading tasks in namespace")
+		if err := m.loadTasks(namespaces.WithNamespace(ctx, ns)); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *TaskManager) loadTasks(ctx context.Context) error {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return err
+	}
+	shimDirs, err := ioutil.ReadDir(filepath.Join(m.state, ns))
+	if err != nil {
+		return err
+	}
+	for _, sd := range shimDirs {
+		if !sd.IsDir() {
+			continue
+		}
+		id := sd.Name()
+		bundle, err := LoadBundle(ctx, m.state, id)
+		if err != nil {
+			return err
+		}
+		shim, err := loadShim(ctx, bundle, m.events, m.tasks)
+		if err != nil {
+			log.G(ctx).WithError(err).Errorf("cleanup dead shim %s", id)
+			container, err := m.container(ctx, id)
+			if err != nil {
+				log.G(ctx).WithError(err).Errorf("loading dead container %s", id)
+				if err := mount.UnmountAll(filepath.Join(bundle.Path, "rootfs"), 0); err != nil {
+					log.G(ctx).WithError(err).Errorf("forceful unmount of rootfs %s", id)
+				}
+				bundle.Delete()
+				continue
+			}
+			binaryCall := shimBinary(ctx, bundle, container.Runtime.Name, m.containerdAddress, m.events, m.tasks)
+			if _, err := binaryCall.Delete(ctx); err != nil {
+				return errors.Wrapf(err, "remove disk state %s", id)
+			}
+			continue
+		}
+		m.tasks.Add(ctx, shim)
+	}
+	return nil
+}
+
+func (m *TaskManager) container(ctx context.Context, id string) (*containers.Container, error) {
+	var container containers.Container
+	if err := m.db.View(func(tx *bolt.Tx) error {
+		store := metadata.NewContainerStore(tx)
+		var err error
+		container, err = store.Get(ctx, id)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+	return &container, nil
+}

runtime/v2/process.go

@@ -0,0 +1,160 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"context"
+
+	eventstypes "github.com/containerd/containerd/api/events"
+	tasktypes "github.com/containerd/containerd/api/types/task"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/runtime"
+	"github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	"github.com/pkg/errors"
+)
+
+type process struct {
+	id   string
+	shim *shim
+}
+
+func (p *process) ID() string {
+	return p.id
+}
+
+func (p *process) Kill(ctx context.Context, signal uint32, _ bool) error {
+	_, err := p.shim.task.Kill(ctx, &task.KillRequest{
+		Signal: signal,
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (p *process) State(ctx context.Context) (runtime.State, error) {
+	response, err := p.shim.task.State(ctx, &task.StateRequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+	})
+	if err != nil {
+		if errors.Cause(err) != ttrpc.ErrClosed {
+			return runtime.State{}, errdefs.FromGRPC(err)
+		}
+		return runtime.State{}, errdefs.ErrNotFound
+	}
+	var status runtime.Status
+	switch response.Status {
+	case tasktypes.StatusCreated:
+		status = runtime.CreatedStatus
+	case tasktypes.StatusRunning:
+		status = runtime.RunningStatus
+	case tasktypes.StatusStopped:
+		status = runtime.StoppedStatus
+	case tasktypes.StatusPaused:
+		status = runtime.PausedStatus
+	case tasktypes.StatusPausing:
+		status = runtime.PausingStatus
+	}
+	return runtime.State{
+		Pid:        response.Pid,
+		Status:     status,
+		Stdin:      response.Stdin,
+		Stdout:     response.Stdout,
+		Stderr:     response.Stderr,
+		Terminal:   response.Terminal,
+		ExitStatus: response.ExitStatus,
+		ExitedAt:   response.ExitedAt,
+	}, nil
+}
+
+// ResizePty changes the side of the process's PTY to the provided width and height
+func (p *process) ResizePty(ctx context.Context, size runtime.ConsoleSize) error {
+	_, err := p.shim.task.ResizePty(ctx, &task.ResizePtyRequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+		Width:  size.Width,
+		Height: size.Height,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+// CloseIO closes the provided IO pipe for the process
+func (p *process) CloseIO(ctx context.Context) error {
+	_, err := p.shim.task.CloseIO(ctx, &task.CloseIORequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+		Stdin:  true,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+// Start the process
+func (p *process) Start(ctx context.Context) error {
+	response, err := p.shim.task.Start(ctx, &task.StartRequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	p.shim.events.Publish(ctx, runtime.TaskExecStartedEventTopic, &eventstypes.TaskExecStarted{
+		ContainerID: p.shim.ID(),
+		Pid:         response.Pid,
+		ExecID:      p.id,
+	})
+	return nil
+}
+
+// Wait on the process to exit and return the exit status and timestamp
+func (p *process) Wait(ctx context.Context) (*runtime.Exit, error) {
+	response, err := p.shim.task.Wait(ctx, &task.WaitRequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return &runtime.Exit{
+		Timestamp: response.ExitedAt,
+		Status:    response.ExitStatus,
+	}, nil
+}
+
+func (p *process) Delete(ctx context.Context) (*runtime.Exit, error) {
+	response, err := p.shim.task.Delete(ctx, &task.DeleteRequest{
+		ID:     p.shim.ID(),
+		ExecID: p.id,
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return &runtime.Exit{
+		Status:    response.ExitStatus,
+		Timestamp: response.ExitedAt,
+		Pid:       response.Pid,
+	}, nil
+}

runtime/v2/runc/epoll.go

@@ -0,0 +1,123 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package runc
+
+import (
+	"context"
+	"sync"
+
+	"github.com/containerd/cgroups"
+	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/runtime"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+func newOOMEpoller(publisher events.Publisher) (*epoller, error) {
+	fd, err := unix.EpollCreate1(unix.EPOLL_CLOEXEC)
+	if err != nil {
+		return nil, err
+	}
+	return &epoller{
+		fd:        fd,
+		publisher: publisher,
+		set:       make(map[uintptr]*item),
+	}, nil
+}
+
+type epoller struct {
+	mu sync.Mutex
+
+	fd        int
+	publisher events.Publisher
+	set       map[uintptr]*item
+}
+
+type item struct {
+	id string
+	cg cgroups.Cgroup
+}
+
+func (e *epoller) Close() error {
+	return unix.Close(e.fd)
+}
+
+func (e *epoller) run(ctx context.Context) {
+	var events [128]unix.EpollEvent
+	for {
+		n, err := unix.EpollWait(e.fd, events[:], -1)
+		if err != nil {
+			if err == unix.EINTR {
+				continue
+			}
+			logrus.WithError(err).Error("cgroups: epoll wait")
+		}
+		for i := 0; i < n; i++ {
+			e.process(ctx, uintptr(events[i].Fd))
+		}
+	}
+}
+
+func (e *epoller) add(id string, cg cgroups.Cgroup) error {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	fd, err := cg.OOMEventFD()
+	if err != nil {
+		return err
+	}
+	e.set[fd] = &item{
+		id: id,
+		cg: cg,
+	}
+	event := unix.EpollEvent{
+		Fd:     int32(fd),
+		Events: unix.EPOLLHUP | unix.EPOLLIN | unix.EPOLLERR,
+	}
+	return unix.EpollCtl(e.fd, unix.EPOLL_CTL_ADD, int(fd), &event)
+}
+
+func (e *epoller) process(ctx context.Context, fd uintptr) {
+	flush(fd)
+	e.mu.Lock()
+	i, ok := e.set[fd]
+	if !ok {
+		e.mu.Unlock()
+		return
+	}
+	e.mu.Unlock()
+	if i.cg.State() == cgroups.Deleted {
+		e.mu.Lock()
+		delete(e.set, fd)
+		e.mu.Unlock()
+		unix.Close(int(fd))
+		return
+	}
+	if err := e.publisher.Publish(ctx, runtime.TaskOOMEventTopic, &eventstypes.TaskOOM{
+		ContainerID: i.id,
+	}); err != nil {
+		logrus.WithError(err).Error("publish OOM event")
+	}
+}
+
+func flush(fd uintptr) error {
+	var buf [8]byte
+	_, err := unix.Read(int(fd), buf[:])
+	return err
+}

runtime/v2/runc/options/doc.go

@@ -0,0 +1,17 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package options

runtime/v2/runc/options/oci.proto

@@ -0,0 +1,50 @@
+syntax = "proto3";
+
+package containerd.runc.v1;
+
+import weak "gogoproto/gogo.proto";
+
+option go_package = "github.com/containerd/containerd/runtime/v2/runc/options;options";
+
+message Options {
+	// disable pivot root when creating a container
+	bool no_pivot_root = 1;
+	// create a new keyring for the container
+	bool no_new_keyring = 2;
+	// place the shim in a cgroup
+	string shim_cgroup = 3;
+	// set the I/O's pipes uid
+	uint32 io_uid = 4;
+	// set the I/O's pipes gid
+	uint32 io_gid = 5;
+	// binary name of the runc binary
+	string binary_name = 6;
+	// runc root directory
+	string root = 7;
+	// criu binary path
+	string criu_path = 8;
+	// enable systemd cgroups
+	bool systemd_cgroup = 9;
+}
+
+message CheckpointOptions {
+	// exit the container after a checkpoint
+	bool exit = 1;
+	// checkpoint open tcp connections
+	bool open_tcp = 2;
+	// checkpoint external unix sockets
+	bool external_unix_sockets = 3;
+	// checkpoint terminals (ptys)
+	bool terminal = 4;
+	// allow checkpointing of file locks
+	bool file_locks = 5;
+	// restore provided namespaces as empty namespaces
+	repeated string empty_namespaces = 6;
+	// set the cgroups mode, soft, full, strict
+	string cgroups_mode = 7;
+}
+
+message ProcessDetails {
+	// exec process id if the process is managed by a shim
+	string exec_id = 1;
+}

runtime/v2/runc/service.go

@@ -0,0 +1,771 @@
+// +build linux
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package runc
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/containerd/cgroups"
+	"github.com/containerd/console"
+	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/api/types/task"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/mount"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/containerd/runtime"
+	rproc "github.com/containerd/containerd/runtime/proc"
+	"github.com/containerd/containerd/runtime/v1/linux/proc"
+	"github.com/containerd/containerd/runtime/v2/runc/options"
+	"github.com/containerd/containerd/runtime/v2/shim"
+	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+	runcC "github.com/containerd/go-runc"
+	"github.com/containerd/typeurl"
+	ptypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+var (
+	empty   = &ptypes.Empty{}
+	bufPool = sync.Pool{
+		New: func() interface{} {
+			buffer := make([]byte, 32<<10)
+			return &buffer
+		},
+	}
+)
+
+var _ = (taskAPI.TaskService)(&service{})
+
+// New returns a new shim service that can be used via GRPC
+func New(ctx context.Context, id string, publisher events.Publisher) (shim.Shim, error) {
+	ep, err := newOOMEpoller(publisher)
+	if err != nil {
+		return nil, err
+	}
+	go ep.run(ctx)
+	s := &service{
+		id:        id,
+		context:   ctx,
+		processes: make(map[string]rproc.Process),
+		events:    make(chan interface{}, 128),
+		ec:        shim.Default.Subscribe(),
+		ep:        ep,
+	}
+	go s.processExits()
+	runcC.Monitor = shim.Default
+	if err := s.initPlatform(); err != nil {
+		return nil, errors.Wrap(err, "failed to initialized platform behavior")
+	}
+	go s.forward(publisher)
+	return s, nil
+}
+
+// service is the shim implementation of a remote shim over GRPC
+type service struct {
+	mu sync.Mutex
+
+	context   context.Context
+	task      rproc.Process
+	processes map[string]rproc.Process
+	events    chan interface{}
+	platform  rproc.Platform
+	ec        chan runcC.Exit
+	ep        *epoller
+
+	id string
+	// Filled by Create()
+	bundle string
+	cg     cgroups.Cgroup
+}
+
+func newCommand(ctx context.Context, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	self, err := os.Executable()
+	if err != nil {
+		return nil, err
+	}
+	cwd, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	args := []string{
+		"-namespace", ns,
+		"-address", containerdAddress,
+		"-publish-binary", containerdBinary,
+	}
+	cmd := exec.Command(self, args...)
+	cmd.Dir = cwd
+	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	return cmd, nil
+}
+
+func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error) {
+	cmd, err := newCommand(ctx, containerdBinary, containerdAddress)
+	if err != nil {
+		return "", err
+	}
+	address, err := shim.AbstractAddress(ctx, id)
+	if err != nil {
+		return "", err
+	}
+	socket, err := shim.NewSocket(address)
+	if err != nil {
+		return "", err
+	}
+	defer socket.Close()
+	f, err := socket.File()
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, f)
+
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+	defer func() {
+		if err != nil {
+			cmd.Process.Kill()
+		}
+	}()
+	// make sure to wait after start
+	go cmd.Wait()
+	if err := shim.WritePidFile("shim.pid", cmd.Process.Pid); err != nil {
+		return "", err
+	}
+	if err := shim.WriteAddress("address", address); err != nil {
+		return "", err
+	}
+	if err := shim.SetScore(cmd.Process.Pid); err != nil {
+		return "", errors.Wrap(err, "failed to set OOM Score on shim")
+	}
+	return address, nil
+}
+
+func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error) {
+	path, err := os.Getwd()
+	if err != nil {
+		return nil, err
+	}
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	runtime, _ := s.readRuntime(path)
+	if runtime != "" {
+		r := proc.NewRunc(proc.RuncRoot, path, ns, runtime, "", false)
+		if err := r.Delete(ctx, s.id, &runcC.DeleteOpts{
+			Force: true,
+		}); err != nil {
+			logrus.WithError(err).Warn("runc delete")
+		}
+	}
+	if err := mount.UnmountAll(filepath.Join(path, "rootfs"), 0); err != nil {
+		logrus.WithError(err).Warn("failed to cleanup rootfs mount")
+	}
+	return &taskAPI.DeleteResponse{
+		ExitedAt:   time.Now(),
+		ExitStatus: 128 + uint32(unix.SIGKILL),
+	}, nil
+}
+
+func (s *service) readRuntime(path string) (string, error) {
+	data, err := ioutil.ReadFile(filepath.Join(path, "runtime"))
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func (s *service) writeRuntime(path, runtime string) error {
+	return ioutil.WriteFile(filepath.Join(path, "runtime"), []byte(runtime), 0600)
+}
+
+// Create a new initial process and container with the underlying OCI runtime
+func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *taskAPI.CreateTaskResponse, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "create namespace")
+	}
+
+	var opts options.Options
+	if r.Options != nil {
+		v, err := typeurl.UnmarshalAny(r.Options)
+		if err != nil {
+			return nil, err
+		}
+		opts = *v.(*options.Options)
+	}
+
+	var mounts []proc.Mount
+	for _, m := range r.Rootfs {
+		mounts = append(mounts, proc.Mount{
+			Type:    m.Type,
+			Source:  m.Source,
+			Target:  m.Target,
+			Options: m.Options,
+		})
+	}
+	config := &proc.CreateConfig{
+		ID:               r.ID,
+		Bundle:           r.Bundle,
+		Runtime:          opts.BinaryName,
+		Rootfs:           mounts,
+		Terminal:         r.Terminal,
+		Stdin:            r.Stdin,
+		Stdout:           r.Stdout,
+		Stderr:           r.Stderr,
+		Checkpoint:       r.Checkpoint,
+		ParentCheckpoint: r.ParentCheckpoint,
+		Options:          r.Options,
+	}
+	if err := s.writeRuntime(r.Bundle, opts.BinaryName); err != nil {
+		return nil, err
+	}
+	rootfs := filepath.Join(r.Bundle, "rootfs")
+	defer func() {
+		if err != nil {
+			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+				logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
+			}
+		}
+	}()
+	for _, rm := range mounts {
+		m := &mount.Mount{
+			Type:    rm.Type,
+			Source:  rm.Source,
+			Options: rm.Options,
+		}
+		if err := m.Mount(rootfs); err != nil {
+			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
+		}
+	}
+	process, err := newInit(
+		ctx,
+		r.Bundle,
+		filepath.Join(r.Bundle, "work"),
+		ns,
+		s.platform,
+		config,
+		&opts,
+	)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	if err := process.Create(ctx, config); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	// save the main task id and bundle to the shim for additional requests
+	s.id = r.ID
+	s.bundle = r.Bundle
+	pid := process.Pid()
+	if pid > 0 {
+		cg, err := cgroups.Load(cgroups.V1, cgroups.PidPath(pid))
+		if err != nil {
+			logrus.WithError(err).Errorf("loading cgroup for %d", pid)
+		}
+		s.setCgroup(cg)
+	}
+	s.task = process
+	return &taskAPI.CreateTaskResponse{
+		Pid: uint32(pid),
+	}, nil
+
+}
+
+// Start a process
+func (s *service) Start(ctx context.Context, r *taskAPI.StartRequest) (*taskAPI.StartResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	if err := p.Start(ctx); err != nil {
+		return nil, err
+	}
+	if s.cg == nil && p.Pid() > 0 {
+		cg, err := cgroups.Load(cgroups.V1, cgroups.PidPath(p.Pid()))
+		if err != nil {
+			logrus.WithError(err).Errorf("loading cgroup for %d", p.Pid())
+		}
+		s.setCgroup(cg)
+	}
+	return &taskAPI.StartResponse{
+		Pid: uint32(p.Pid()),
+	}, nil
+}
+
+// Delete the initial process and container
+func (s *service) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAPI.DeleteResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	if err := p.Delete(ctx); err != nil {
+		return nil, err
+	}
+	isTask := r.ExecID == ""
+	if !isTask {
+		delete(s.processes, r.ExecID)
+	}
+	if isTask && s.platform != nil {
+		s.platform.Close()
+	}
+	return &taskAPI.DeleteResponse{
+		ExitStatus: uint32(p.ExitStatus()),
+		ExitedAt:   p.ExitedAt(),
+		Pid:        uint32(p.Pid()),
+	}, nil
+}
+
+// Exec an additional process inside the container
+func (s *service) Exec(ctx context.Context, r *taskAPI.ExecProcessRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if p := s.processes[r.ExecID]; p != nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrAlreadyExists, "id %s", r.ExecID)
+	}
+	p := s.task
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	process, err := p.(*proc.Init).Exec(ctx, s.bundle, &proc.ExecConfig{
+		ID:       r.ExecID,
+		Terminal: r.Terminal,
+		Stdin:    r.Stdin,
+		Stdout:   r.Stdout,
+		Stderr:   r.Stderr,
+		Spec:     r.Spec,
+	})
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	s.processes[r.ExecID] = process
+	return empty, nil
+}
+
+// ResizePty of a process
+func (s *service) ResizePty(ctx context.Context, r *taskAPI.ResizePtyRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	ws := console.WinSize{
+		Width:  uint16(r.Width),
+		Height: uint16(r.Height),
+	}
+	if err := p.Resize(ws); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return empty, nil
+}
+
+// State returns runtime state information for a process
+func (s *service) State(ctx context.Context, r *taskAPI.StateRequest) (*taskAPI.StateResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	st, err := p.Status(ctx)
+	if err != nil {
+		return nil, err
+	}
+	status := task.StatusUnknown
+	switch st {
+	case "created":
+		status = task.StatusCreated
+	case "running":
+		status = task.StatusRunning
+	case "stopped":
+		status = task.StatusStopped
+	case "paused":
+		status = task.StatusPaused
+	case "pausing":
+		status = task.StatusPausing
+	}
+	sio := p.Stdio()
+	return &taskAPI.StateResponse{
+		ID:         p.ID(),
+		Bundle:     s.bundle,
+		Pid:        uint32(p.Pid()),
+		Status:     status,
+		Stdin:      sio.Stdin,
+		Stdout:     sio.Stdout,
+		Stderr:     sio.Stderr,
+		Terminal:   sio.Terminal,
+		ExitStatus: uint32(p.ExitStatus()),
+		ExitedAt:   p.ExitedAt(),
+	}, nil
+}
+
+// Pause the container
+func (s *service) Pause(ctx context.Context, r *taskAPI.PauseRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p := s.task
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	if err := p.(*proc.Init).Pause(ctx); err != nil {
+		return nil, err
+	}
+	return empty, nil
+}
+
+// Resume the container
+func (s *service) Resume(ctx context.Context, r *taskAPI.ResumeRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p := s.task
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	if err := p.(*proc.Init).Resume(ctx); err != nil {
+		return nil, err
+	}
+	return empty, nil
+}
+
+// Kill a process with the provided signal
+func (s *service) Kill(ctx context.Context, r *taskAPI.KillRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	if err := p.Kill(ctx, r.Signal, r.All); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return empty, nil
+}
+
+// Pids returns all pids inside the container
+func (s *service) Pids(ctx context.Context, r *taskAPI.PidsRequest) (*taskAPI.PidsResponse, error) {
+	pids, err := s.getContainerPids(ctx, r.ID)
+	if err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	var processes []*task.ProcessInfo
+	for _, pid := range pids {
+		pInfo := task.ProcessInfo{
+			Pid: pid,
+		}
+		for _, p := range s.processes {
+			if p.Pid() == int(pid) {
+				d := &options.ProcessDetails{
+					ExecID: p.ID(),
+				}
+				a, err := typeurl.MarshalAny(d)
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to marshal process %d info", pid)
+				}
+				pInfo.Info = a
+				break
+			}
+		}
+		processes = append(processes, &pInfo)
+	}
+	return &taskAPI.PidsResponse{
+		Processes: processes,
+	}, nil
+}
+
+// CloseIO of a process
+func (s *service) CloseIO(ctx context.Context, r *taskAPI.CloseIORequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p, err := s.getProcess(r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	if stdin := p.Stdin(); stdin != nil {
+		if err := stdin.Close(); err != nil {
+			return nil, errors.Wrap(err, "close stdin")
+		}
+	}
+	return empty, nil
+}
+
+// Checkpoint the container
+func (s *service) Checkpoint(ctx context.Context, r *taskAPI.CheckpointTaskRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p := s.task
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	var opts options.CheckpointOptions
+	if r.Options != nil {
+		v, err := typeurl.UnmarshalAny(r.Options)
+		if err != nil {
+			return nil, err
+		}
+		opts = *v.(*options.CheckpointOptions)
+	}
+	if err := p.(*proc.Init).Checkpoint(ctx, &proc.CheckpointConfig{
+		Path:                     r.Path,
+		Exit:                     opts.Exit,
+		AllowOpenTCP:             opts.OpenTcp,
+		AllowExternalUnixSockets: opts.ExternalUnixSockets,
+		AllowTerminal:            opts.Terminal,
+		FileLocks:                opts.FileLocks,
+		EmptyNamespaces:          opts.EmptyNamespaces,
+	}); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return empty, nil
+}
+
+// Connect returns shim information such as the shim's pid
+func (s *service) Connect(ctx context.Context, r *taskAPI.ConnectRequest) (*taskAPI.ConnectResponse, error) {
+	var pid int
+	if s.task != nil {
+		pid = s.task.Pid()
+	}
+	return &taskAPI.ConnectResponse{
+		ShimPid: uint32(os.Getpid()),
+		TaskPid: uint32(pid),
+	}, nil
+}
+
+func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (*ptypes.Empty, error) {
+	os.Exit(0)
+	return empty, nil
+}
+
+func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (*taskAPI.StatsResponse, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	if s.cg == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "cgroup does not exist")
+	}
+	stats, err := s.cg.Stat(cgroups.IgnoreNotExist)
+	if err != nil {
+		return nil, err
+	}
+	data, err := typeurl.MarshalAny(stats)
+	if err != nil {
+		return nil, err
+	}
+	return &taskAPI.StatsResponse{
+		Stats: data,
+	}, nil
+}
+
+// Update a running container
+func (s *service) Update(ctx context.Context, r *taskAPI.UpdateTaskRequest) (*ptypes.Empty, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p := s.task
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	if err := p.(*proc.Init).Update(ctx, r.Resources); err != nil {
+		return nil, errdefs.ToGRPC(err)
+	}
+	return empty, nil
+}
+
+// Wait for a process to exit
+func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (*taskAPI.WaitResponse, error) {
+	s.mu.Lock()
+	p, err := s.getProcess(r.ExecID)
+	s.mu.Unlock()
+	if err != nil {
+		return nil, err
+	}
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	p.Wait()
+
+	return &taskAPI.WaitResponse{
+		ExitStatus: uint32(p.ExitStatus()),
+		ExitedAt:   p.ExitedAt(),
+	}, nil
+}
+
+func (s *service) processExits() {
+	for e := range s.ec {
+		s.checkProcesses(e)
+	}
+}
+
+func (s *service) checkProcesses(e runcC.Exit) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	for _, p := range s.allProcesses() {
+		if p.Pid() == e.Pid {
+			if ip, ok := p.(*proc.Init); ok {
+				// Ensure all children are killed
+				if err := ip.KillAll(s.context); err != nil {
+					logrus.WithError(err).WithField("id", ip.ID()).
+						Error("failed to kill init's children")
+				}
+			}
+			p.SetExited(e.Status)
+			s.events <- &eventstypes.TaskExit{
+				ContainerID: s.id,
+				ID:          p.ID(),
+				Pid:         uint32(e.Pid),
+				ExitStatus:  uint32(e.Status),
+				ExitedAt:    p.ExitedAt(),
+			}
+			return
+		}
+	}
+}
+
+func (s *service) allProcesses() (o []rproc.Process) {
+	for _, p := range s.processes {
+		o = append(o, p)
+	}
+	if s.task != nil {
+		o = append(o, s.task)
+	}
+	return o
+}
+
+func (s *service) getContainerPids(ctx context.Context, id string) ([]uint32, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	p := s.task
+	if p == nil {
+		return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "container must be created")
+	}
+	ps, err := p.(*proc.Init).Runtime().Ps(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	pids := make([]uint32, 0, len(ps))
+	for _, pid := range ps {
+		pids = append(pids, uint32(pid))
+	}
+	return pids, nil
+}
+
+func (s *service) forward(publisher events.Publisher) {
+	for e := range s.events {
+		if err := publisher.Publish(s.context, getTopic(s.context, e), e); err != nil {
+			logrus.WithError(err).Error("post event")
+		}
+	}
+}
+
+func (s *service) getProcess(execID string) (rproc.Process, error) {
+	if execID == "" {
+		return s.task, nil
+	}
+	p := s.processes[execID]
+	if p == nil {
+		return nil, errdefs.ToGRPCf(errdefs.ErrNotFound, "process does not exist %s", execID)
+	}
+	return p, nil
+}
+
+func (s *service) setCgroup(cg cgroups.Cgroup) {
+	s.cg = cg
+	if err := s.ep.add(s.id, cg); err != nil {
+		logrus.WithError(err).Error("add cg to OOM monitor")
+	}
+}
+
+func getTopic(ctx context.Context, e interface{}) string {
+	switch e.(type) {
+	case *eventstypes.TaskCreate:
+		return runtime.TaskCreateEventTopic
+	case *eventstypes.TaskStart:
+		return runtime.TaskStartEventTopic
+	case *eventstypes.TaskOOM:
+		return runtime.TaskOOMEventTopic
+	case *eventstypes.TaskExit:
+		return runtime.TaskExitEventTopic
+	case *eventstypes.TaskDelete:
+		return runtime.TaskDeleteEventTopic
+	case *eventstypes.TaskExecAdded:
+		return runtime.TaskExecAddedEventTopic
+	case *eventstypes.TaskExecStarted:
+		return runtime.TaskExecStartedEventTopic
+	case *eventstypes.TaskPaused:
+		return runtime.TaskPausedEventTopic
+	case *eventstypes.TaskResumed:
+		return runtime.TaskResumedEventTopic
+	case *eventstypes.TaskCheckpointed:
+		return runtime.TaskCheckpointedEventTopic
+	default:
+		logrus.Warnf("no topic for type %#v", e)
+	}
+	return runtime.TaskUnknownTopic
+}
+
+func newInit(ctx context.Context, path, workDir, namespace string, platform rproc.Platform, r *proc.CreateConfig, options *options.Options) (*proc.Init, error) {
+	rootfs := filepath.Join(path, "rootfs")
+	runtime := proc.NewRunc(options.Root, path, namespace, options.BinaryName, options.CriuPath, options.SystemdCgroup)
+	p := proc.New(r.ID, runtime, rproc.Stdio{
+		Stdin:    r.Stdin,
+		Stdout:   r.Stdout,
+		Stderr:   r.Stderr,
+		Terminal: r.Terminal,
+	})
+	p.Bundle = r.Bundle
+	p.Platform = platform
+	p.Rootfs = rootfs
+	p.WorkDir = workDir
+	p.IoUID = int(options.IoUid)
+	p.IoGID = int(options.IoGid)
+	p.NoPivotRoot = options.NoPivotRoot
+	p.NoNewKeyring = options.NoNewKeyring
+	return p, nil
+}

runtime/v2/runc/service_linux.go

@@ -0,0 +1,111 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package runc
+
+import (
+	"context"
+	"io"
+	"sync"
+	"syscall"
+
+	"github.com/containerd/console"
+	"github.com/containerd/fifo"
+	"github.com/pkg/errors"
+)
+
+type linuxPlatform struct {
+	epoller *console.Epoller
+}
+
+func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console, stdin, stdout, stderr string, wg, cwg *sync.WaitGroup) (console.Console, error) {
+	if p.epoller == nil {
+		return nil, errors.New("uninitialized epoller")
+	}
+
+	epollConsole, err := p.epoller.Add(console)
+	if err != nil {
+		return nil, err
+	}
+
+	if stdin != "" {
+		in, err := fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY, 0)
+		if err != nil {
+			return nil, err
+		}
+		cwg.Add(1)
+		go func() {
+			cwg.Done()
+			p := bufPool.Get().(*[]byte)
+			defer bufPool.Put(p)
+			io.CopyBuffer(epollConsole, in, *p)
+		}()
+	}
+
+	outw, err := fifo.OpenFifo(ctx, stdout, syscall.O_WRONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	outr, err := fifo.OpenFifo(ctx, stdout, syscall.O_RDONLY, 0)
+	if err != nil {
+		return nil, err
+	}
+	wg.Add(1)
+	cwg.Add(1)
+	go func() {
+		cwg.Done()
+		p := bufPool.Get().(*[]byte)
+		defer bufPool.Put(p)
+		io.CopyBuffer(outw, epollConsole, *p)
+		epollConsole.Close()
+		outr.Close()
+		outw.Close()
+		wg.Done()
+	}()
+	return epollConsole, nil
+}
+
+func (p *linuxPlatform) ShutdownConsole(ctx context.Context, cons console.Console) error {
+	if p.epoller == nil {
+		return errors.New("uninitialized epoller")
+	}
+	epollConsole, ok := cons.(*console.EpollConsole)
+	if !ok {
+		return errors.Errorf("expected EpollConsole, got %#v", cons)
+	}
+	return epollConsole.Shutdown(p.epoller.CloseConsole)
+}
+
+func (p *linuxPlatform) Close() error {
+	return p.epoller.Close()
+}
+
+// initialize a single epoll fd to manage our consoles. `initPlatform` should
+// only be called once.
+func (s *service) initPlatform() error {
+	if s.platform != nil {
+		return nil
+	}
+	epoller, err := console.NewEpoller()
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize epoller")
+	}
+	s.platform = &linuxPlatform{
+		epoller: epoller,
+	}
+	go epoller.Wait()
+	return nil
+}

runtime/v2/shim.go

@@ -0,0 +1,388 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package v2
+
+import (
+	"context"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	eventstypes "github.com/containerd/containerd/api/events"
+	"github.com/containerd/containerd/api/types"
+	tasktypes "github.com/containerd/containerd/api/types/task"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events/exchange"
+	"github.com/containerd/containerd/identifiers"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/runtime"
+	client "github.com/containerd/containerd/runtime/v2/shim"
+	"github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	ptypes "github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+)
+
+func loadAddress(path string) (string, error) {
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	return string(data), nil
+}
+
+func loadShim(ctx context.Context, bundle *Bundle, events *exchange.Exchange, rt *runtime.TaskList) (_ *shim, err error) {
+	address, err := loadAddress(filepath.Join(bundle.Path, "address"))
+	if err != nil {
+		return nil, err
+	}
+	conn, err := client.Connect(address, client.AnonDialer)
+	if err != nil {
+		return nil, err
+	}
+	client := ttrpc.NewClient(conn)
+	client.OnClose(func() { conn.Close() })
+	s := &shim{
+		client:  client,
+		task:    task.NewTaskClient(client),
+		bundle:  bundle,
+		events:  events,
+		rtTasks: rt,
+	}
+	if err := s.Connect(ctx); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+type shim struct {
+	bundle  *Bundle
+	client  *ttrpc.Client
+	task    task.TaskService
+	taskPid int
+	events  *exchange.Exchange
+	rtTasks *runtime.TaskList
+}
+
+func (s *shim) Connect(ctx context.Context) error {
+	response, err := s.task.Connect(ctx, &task.ConnectRequest{})
+	if err != nil {
+		return err
+	}
+	s.taskPid = int(response.TaskPid)
+	return nil
+}
+
+func (s *shim) Shutdown(ctx context.Context) error {
+	_, err := s.task.Shutdown(ctx, &task.ShutdownRequest{
+		ID: s.ID(),
+	})
+	if err != nil && err != ttrpc.ErrClosed {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (s *shim) waitShutdown(ctx context.Context) error {
+	dead := make(chan struct{})
+	go func() {
+		if err := s.Shutdown(ctx); err != nil {
+			log.G(ctx).WithError(err).Error("shim shutdown error")
+		}
+		close(dead)
+	}()
+	select {
+	case <-time.After(3 * time.Second):
+		return errors.New("failed to shutdown shim in time")
+	case <-dead:
+		return nil
+	}
+}
+
+// ID of the shim/task
+func (s *shim) ID() string {
+	return s.bundle.ID
+}
+
+func (s *shim) Namespace() string {
+	return s.bundle.Namespace
+}
+
+func (s *shim) Close() error {
+	return s.client.Close()
+}
+
+func (s *shim) Delete(ctx context.Context) (*runtime.Exit, error) {
+	response, err := s.task.Delete(ctx, &task.DeleteRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	if err := s.waitShutdown(ctx); err != nil {
+		return nil, err
+	}
+	if err := s.bundle.Delete(); err != nil {
+		return nil, err
+	}
+	// remove self from the runtime task list
+	// this seems dirty but it cleans up the API across runtimes, tasks, and the service
+	s.rtTasks.Delete(ctx, s.ID())
+	s.events.Publish(ctx, runtime.TaskDeleteEventTopic, &eventstypes.TaskDelete{
+		ContainerID: s.ID(),
+		ExitStatus:  response.ExitStatus,
+		ExitedAt:    response.ExitedAt,
+		Pid:         response.Pid,
+	})
+	return &runtime.Exit{
+		Status:    response.ExitStatus,
+		Timestamp: response.ExitedAt,
+		Pid:       response.Pid,
+	}, nil
+}
+
+func (s *shim) Create(ctx context.Context, opts runtime.CreateOpts) (runtime.Task, error) {
+	topts := opts.TaskOptions
+	if topts == nil {
+		topts = opts.RuntimeOptions
+	}
+	request := &task.CreateTaskRequest{
+		ID:         s.ID(),
+		Bundle:     s.bundle.Path,
+		Stdin:      opts.IO.Stdin,
+		Stdout:     opts.IO.Stdout,
+		Stderr:     opts.IO.Stderr,
+		Terminal:   opts.IO.Terminal,
+		Checkpoint: opts.Checkpoint,
+		Options:    topts,
+	}
+	for _, m := range opts.Rootfs {
+		request.Rootfs = append(request.Rootfs, &types.Mount{
+			Type:    m.Type,
+			Source:  m.Source,
+			Options: m.Options,
+		})
+	}
+	response, err := s.task.Create(ctx, request)
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	s.taskPid = int(response.Pid)
+	return s, nil
+}
+
+func (s *shim) Pause(ctx context.Context) error {
+	if _, err := s.task.Pause(ctx, &task.PauseRequest{
+		ID: s.ID(),
+	}); err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	s.events.Publish(ctx, runtime.TaskPausedEventTopic, &eventstypes.TaskPaused{
+		ContainerID: s.ID(),
+	})
+	return nil
+}
+
+func (s *shim) Resume(ctx context.Context) error {
+	if _, err := s.task.Resume(ctx, &task.ResumeRequest{
+		ID: s.ID(),
+	}); err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	s.events.Publish(ctx, runtime.TaskResumedEventTopic, &eventstypes.TaskResumed{
+		ContainerID: s.ID(),
+	})
+	return nil
+}
+
+func (s *shim) Start(ctx context.Context) error {
+	response, err := s.task.Start(ctx, &task.StartRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	s.taskPid = int(response.Pid)
+	s.events.Publish(ctx, runtime.TaskStartEventTopic, &eventstypes.TaskStart{
+		ContainerID: s.ID(),
+		Pid:         uint32(s.taskPid),
+	})
+	return nil
+}
+
+func (s *shim) Kill(ctx context.Context, signal uint32, all bool) error {
+	if _, err := s.task.Kill(ctx, &task.KillRequest{
+		ID:     s.ID(),
+		Signal: signal,
+		All:    all,
+	}); err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (s *shim) Exec(ctx context.Context, id string, opts runtime.ExecOpts) (runtime.Process, error) {
+	if err := identifiers.Validate(id); err != nil {
+		return nil, errors.Wrapf(err, "invalid exec id %s", id)
+	}
+	request := &task.ExecProcessRequest{
+		ID:       s.ID(),
+		ExecID:   id,
+		Stdin:    opts.IO.Stdin,
+		Stdout:   opts.IO.Stdout,
+		Stderr:   opts.IO.Stderr,
+		Terminal: opts.IO.Terminal,
+		Spec:     opts.Spec,
+	}
+	if _, err := s.task.Exec(ctx, request); err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return &process{
+		id:   id,
+		shim: s,
+	}, nil
+}
+
+func (s *shim) Pids(ctx context.Context) ([]runtime.ProcessInfo, error) {
+	resp, err := s.task.Pids(ctx, &task.PidsRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	var processList []runtime.ProcessInfo
+	for _, p := range resp.Processes {
+		processList = append(processList, runtime.ProcessInfo{
+			Pid:  p.Pid,
+			Info: p.Info,
+		})
+	}
+	return processList, nil
+}
+
+func (s *shim) ResizePty(ctx context.Context, size runtime.ConsoleSize) error {
+	_, err := s.task.ResizePty(ctx, &task.ResizePtyRequest{
+		ID:     s.ID(),
+		Width:  size.Width,
+		Height: size.Height,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (s *shim) CloseIO(ctx context.Context) error {
+	_, err := s.task.CloseIO(ctx, &task.CloseIORequest{
+		ID:    s.ID(),
+		Stdin: true,
+	})
+	if err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (s *shim) Wait(ctx context.Context) (*runtime.Exit, error) {
+	response, err := s.task.Wait(ctx, &task.WaitRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return &runtime.Exit{
+		Pid:       uint32(s.taskPid),
+		Timestamp: response.ExitedAt,
+		Status:    response.ExitStatus,
+	}, nil
+}
+
+func (s *shim) Checkpoint(ctx context.Context, path string, options *ptypes.Any) error {
+	request := &task.CheckpointTaskRequest{
+		Path:    path,
+		Options: options,
+	}
+	if _, err := s.task.Checkpoint(ctx, request); err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	s.events.Publish(ctx, runtime.TaskCheckpointedEventTopic, &eventstypes.TaskCheckpointed{
+		ContainerID: s.ID(),
+	})
+	return nil
+}
+
+func (s *shim) Update(ctx context.Context, resources *ptypes.Any) error {
+	if _, err := s.task.Update(ctx, &task.UpdateTaskRequest{
+		ID:        s.ID(),
+		Resources: resources,
+	}); err != nil {
+		return errdefs.FromGRPC(err)
+	}
+	return nil
+}
+
+func (s *shim) Stats(ctx context.Context) (*ptypes.Any, error) {
+	response, err := s.task.Stats(ctx, &task.StatsRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		return nil, errdefs.FromGRPC(err)
+	}
+	return response.Stats, nil
+}
+
+func (s *shim) Process(ctx context.Context, id string) (runtime.Process, error) {
+	return &process{
+		id:   id,
+		shim: s,
+	}, nil
+}
+
+func (s *shim) State(ctx context.Context) (runtime.State, error) {
+	response, err := s.task.State(ctx, &task.StateRequest{
+		ID: s.ID(),
+	})
+	if err != nil {
+		if errors.Cause(err) != ttrpc.ErrClosed {
+			return runtime.State{}, errdefs.FromGRPC(err)
+		}
+		return runtime.State{}, errdefs.ErrNotFound
+	}
+	var status runtime.Status
+	switch response.Status {
+	case tasktypes.StatusCreated:
+		status = runtime.CreatedStatus
+	case tasktypes.StatusRunning:
+		status = runtime.RunningStatus
+	case tasktypes.StatusStopped:
+		status = runtime.StoppedStatus
+	case tasktypes.StatusPaused:
+		status = runtime.PausedStatus
+	case tasktypes.StatusPausing:
+		status = runtime.PausingStatus
+	}
+	return runtime.State{
+		Pid:        response.Pid,
+		Status:     status,
+		Stdin:      response.Stdin,
+		Stdout:     response.Stdout,
+		Stderr:     response.Stderr,
+		Terminal:   response.Terminal,
+		ExitStatus: response.ExitStatus,
+		ExitedAt:   response.ExitedAt,
+	}, nil
+}

runtime/v2/shim/reaper_unix.go

@@ -0,0 +1,110 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"os/exec"
+	"sync"
+	"time"
+
+	"github.com/containerd/containerd/sys"
+	runc "github.com/containerd/go-runc"
+	"github.com/pkg/errors"
+)
+
+// ErrNoSuchProcess is returned when the process no longer exists
+var ErrNoSuchProcess = errors.New("no such process")
+
+const bufferSize = 32
+
+// Reap should be called when the process receives an SIGCHLD.  Reap will reap
+// all exited processes and close their wait channels
+func Reap() error {
+	now := time.Now()
+	exits, err := sys.Reap(false)
+	Default.Lock()
+	for c := range Default.subscribers {
+		for _, e := range exits {
+			c <- runc.Exit{
+				Timestamp: now,
+				Pid:       e.Pid,
+				Status:    e.Status,
+			}
+		}
+
+	}
+	Default.Unlock()
+	return err
+}
+
+// Default is the default monitor initialized for the package
+var Default = &Monitor{
+	subscribers: make(map[chan runc.Exit]struct{}),
+}
+
+// Monitor monitors the underlying system for process status changes
+type Monitor struct {
+	sync.Mutex
+
+	subscribers map[chan runc.Exit]struct{}
+}
+
+// Start starts the command a registers the process with the reaper
+func (m *Monitor) Start(c *exec.Cmd) (chan runc.Exit, error) {
+	ec := m.Subscribe()
+	if err := c.Start(); err != nil {
+		m.Unsubscribe(ec)
+		return nil, err
+	}
+	return ec, nil
+}
+
+// Wait blocks until a process is signal as dead.
+// User should rely on the value of the exit status to determine if the
+// command was successful or not.
+func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) {
+	for e := range ec {
+		if e.Pid == c.Process.Pid {
+			// make sure we flush all IO
+			c.Wait()
+			m.Unsubscribe(ec)
+			return e.Status, nil
+		}
+	}
+	// return no such process if the ec channel is closed and no more exit
+	// events will be sent
+	return -1, ErrNoSuchProcess
+}
+
+// Subscribe to process exit changes
+func (m *Monitor) Subscribe() chan runc.Exit {
+	c := make(chan runc.Exit, bufferSize)
+	m.Lock()
+	m.subscribers[c] = struct{}{}
+	m.Unlock()
+	return c
+}
+
+// Unsubscribe to process exit changes
+func (m *Monitor) Unsubscribe(c chan runc.Exit) {
+	m.Lock()
+	delete(m.subscribers, c)
+	close(c)
+	m.Unlock()
+}

runtime/v2/shim/shim.go

@@ -0,0 +1,300 @@
+// +build !windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"os/signal"
+	"runtime"
+	"runtime/debug"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/namespaces"
+	shimapi "github.com/containerd/containerd/runtime/v2/task"
+	"github.com/containerd/ttrpc"
+	"github.com/containerd/typeurl"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// Client for a shim server
+type Client struct {
+	service shimapi.TaskService
+	context context.Context
+	signals chan os.Signal
+}
+
+// Init func for the creation of a shim server
+type Init func(context.Context, string, events.Publisher) (Shim, error)
+
+// Shim server interface
+type Shim interface {
+	shimapi.TaskService
+	Cleanup(ctx context.Context) (*shimapi.DeleteResponse, error)
+	StartShim(ctx context.Context, id, containerdBinary, containerdAddress string) (string, error)
+}
+
+var (
+	debugFlag            bool
+	idFlag               string
+	namespaceFlag        string
+	socketFlag           string
+	addressFlag          string
+	containerdBinaryFlag string
+	action               string
+)
+
+func parseFlags() {
+	flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
+	flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
+	flag.StringVar(&idFlag, "id", "", "id of the task")
+	flag.StringVar(&socketFlag, "socket", "", "abstract socket path to serve")
+
+	flag.StringVar(&addressFlag, "address", "", "grpc address back to main containerd")
+	flag.StringVar(&containerdBinaryFlag, "publish-binary", "containerd", "path to publish binary (used for publishing events)")
+
+	flag.Parse()
+	action = flag.Arg(0)
+}
+
+func setRuntime() {
+	debug.SetGCPercent(40)
+	go func() {
+		for range time.Tick(30 * time.Second) {
+			debug.FreeOSMemory()
+		}
+	}()
+	if debugFlag {
+		f, err := os.OpenFile("shim.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "open shim log %s", err)
+			os.Exit(1)
+		}
+		logrus.SetLevel(logrus.DebugLevel)
+		logrus.SetOutput(f)
+	}
+	if os.Getenv("GOMAXPROCS") == "" {
+		// If GOMAXPROCS hasn't been set, we default to a value of 2 to reduce
+		// the number of Go stacks present in the shim.
+		runtime.GOMAXPROCS(2)
+	}
+}
+
+// Run initializes and runs a shim server
+func Run(initFunc Init) error {
+	parseFlags()
+	setRuntime()
+
+	signals, err := setupSignals()
+	if err != nil {
+		return err
+	}
+	if err := subreaper(); err != nil {
+		return err
+	}
+	publisher := &remoteEventsPublisher{
+		address:              addressFlag,
+		containerdBinaryPath: containerdBinaryFlag,
+	}
+	if namespaceFlag == "" {
+		return fmt.Errorf("shim namespace cannot be empty")
+	}
+	ctx := namespaces.WithNamespace(context.Background(), namespaceFlag)
+	service, err := initFunc(ctx, idFlag, publisher)
+	if err != nil {
+		return err
+	}
+	switch action {
+	case "delete":
+		logger := logrus.WithFields(logrus.Fields{
+			"pid":       os.Getpid(),
+			"namespace": namespaceFlag,
+		})
+		go handleSignals(logger, signals)
+		response, err := service.Cleanup(ctx)
+		if err != nil {
+			return err
+		}
+		data, err := response.Marshal()
+		if err != nil {
+			return err
+		}
+		if _, err := os.Stdout.Write(data); err != nil {
+			return err
+		}
+		return nil
+	case "start":
+		address, err := service.StartShim(ctx, idFlag, containerdBinaryFlag, addressFlag)
+		if err != nil {
+			return err
+		}
+		if _, err := os.Stdout.WriteString(address); err != nil {
+			return err
+		}
+		return nil
+	default:
+		client := NewShimClient(ctx, service, signals)
+		return client.Serve()
+	}
+}
+
+// NewShimClient creates a new shim server client
+func NewShimClient(ctx context.Context, svc shimapi.TaskService, signals chan os.Signal) *Client {
+	s := &Client{
+		service: svc,
+		context: ctx,
+		signals: signals,
+	}
+	return s
+}
+
+// Serve the shim server
+func (s *Client) Serve() error {
+	dump := make(chan os.Signal, 32)
+	signal.Notify(dump, syscall.SIGUSR1)
+
+	path, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+	server, err := newServer()
+	if err != nil {
+		return errors.Wrap(err, "failed creating server")
+	}
+
+	logrus.Debug("registering ttrpc server")
+	shimapi.RegisterTaskService(server, s.service)
+
+	if err := serve(s.context, server, socketFlag); err != nil {
+		return err
+	}
+	logger := logrus.WithFields(logrus.Fields{
+		"pid":       os.Getpid(),
+		"path":      path,
+		"namespace": namespaceFlag,
+	})
+	go func() {
+		for range dump {
+			dumpStacks(logger)
+		}
+	}()
+	return handleSignals(logger, s.signals)
+}
+
+// serve serves the ttrpc API over a unix socket at the provided path
+// this function does not block
+func serve(ctx context.Context, server *ttrpc.Server, path string) error {
+	var (
+		l   net.Listener
+		err error
+	)
+	if path == "" {
+		l, err = net.FileListener(os.NewFile(3, "socket"))
+		path = "[inherited from parent]"
+	} else {
+		if len(path) > 106 {
+			return errors.Errorf("%q: unix socket path too long (> 106)", path)
+		}
+		l, err = net.Listen("unix", "\x00"+path)
+	}
+	if err != nil {
+		return err
+	}
+	logrus.WithField("socket", path).Debug("serving api on unix socket")
+	go func() {
+		defer l.Close()
+		if err := server.Serve(ctx, l); err != nil &&
+			!strings.Contains(err.Error(), "use of closed network connection") {
+			logrus.WithError(err).Fatal("containerd-shim: ttrpc server failure")
+		}
+	}()
+	return nil
+}
+
+func handleSignals(logger *logrus.Entry, signals chan os.Signal) error {
+	logger.Info("starting signal loop")
+	for {
+		select {
+		case s := <-signals:
+			switch s {
+			case unix.SIGCHLD:
+				if err := Reap(); err != nil {
+					logger.WithError(err).Error("reap exit status")
+				}
+			case unix.SIGPIPE:
+			}
+		}
+	}
+}
+
+func dumpStacks(logger *logrus.Entry) {
+	var (
+		buf       []byte
+		stackSize int
+	)
+	bufferLen := 16384
+	for stackSize == len(buf) {
+		buf = make([]byte, bufferLen)
+		stackSize = runtime.Stack(buf, true)
+		bufferLen *= 2
+	}
+	buf = buf[:stackSize]
+	logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf)
+}
+
+type remoteEventsPublisher struct {
+	address              string
+	containerdBinaryPath string
+}
+
+func (l *remoteEventsPublisher) Publish(ctx context.Context, topic string, event events.Event) error {
+	ns, _ := namespaces.Namespace(ctx)
+	encoded, err := typeurl.MarshalAny(event)
+	if err != nil {
+		return err
+	}
+	data, err := encoded.Marshal()
+	if err != nil {
+		return err
+	}
+	cmd := exec.CommandContext(ctx, l.containerdBinaryPath, "--address", l.address, "publish", "--topic", topic, "--namespace", ns)
+	cmd.Stdin = bytes.NewReader(data)
+	c, err := Default.Start(cmd)
+	if err != nil {
+		return err
+	}
+	status, err := Default.Wait(cmd, c)
+	if err != nil {
+		return err
+	}
+	if status != 0 {
+		return errors.New("failed to publish event")
+	}
+	return nil
+}

runtime/v2/shim/shim_darwin.go

@@ -0,0 +1,45 @@
+// +build darwin
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"os"
+	"os/signal"
+
+	"github.com/containerd/ttrpc"
+)
+
+// setupSignals creates a new signal handler for all signals and sets the shim as a
+// sub-reaper so that the container processes are reparented
+func setupSignals() (chan os.Signal, error) {
+	signals := make(chan os.Signal, 2048)
+	signal.Notify(signals)
+	return signals, nil
+}
+
+func newServer() (*ttrpc.Server, error) {
+	// for darwin, we omit the socket credentials because these syscalls are
+	// slightly different. since we don't have darwin support yet, this can be
+	// implemented later and the build can continue without issue.
+	return ttrpc.NewServer()
+}
+
+func subreaper() error {
+	return nil
+}

runtime/v2/shim/shim_linux.go

@@ -0,0 +1,42 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"os"
+	"os/signal"
+
+	"github.com/containerd/containerd/sys"
+	"github.com/containerd/ttrpc"
+	"golang.org/x/sys/unix"
+)
+
+// setupSignals creates a new signal handler for all signals and sets the shim as a
+// sub-reaper so that the container processes are reparented
+func setupSignals() (chan os.Signal, error) {
+	signals := make(chan os.Signal, 32)
+	signal.Notify(signals, unix.SIGTERM, unix.SIGINT, unix.SIGCHLD, unix.SIGPIPE)
+	return signals, nil
+}
+
+func newServer() (*ttrpc.Server, error) {
+	return ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser()))
+}
+
+func subreaper() error {
+	return sys.SetSubreaper(1)
+}

runtime/v2/shim/shim_unix.go

@@ -0,0 +1,42 @@
+// +build !linux,!windows,!darwin
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"os"
+	"os/signal"
+
+	"github.com/containerd/ttrpc"
+)
+
+// setupSignals creates a new signal handler for all signals and sets the shim as a
+// sub-reaper so that the container processes are reparented
+func setupSignals() (chan os.Signal, error) {
+	signals := make(chan os.Signal, 2048)
+	signal.Notify(signals)
+	return signals, nil
+}
+
+func newServer() (*ttrpc.Server, error) {
+	return ttrpc.NewServer(ttrpc.WithServerHandshaker(ttrpc.UnixSocketRequireSameUser()))
+}
+
+func subreaper() error {
+	return nil
+}

runtime/v2/shim/util.go

@@ -0,0 +1,141 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/namespaces"
+	"github.com/pkg/errors"
+)
+
+const shimBinaryFormat = "containerd-shim-%s-%s"
+
+// Command returns the shim command with the provided args and configuration
+func Command(ctx context.Context, runtime, containerdAddress, path string, cmdArgs ...string) (*exec.Cmd, error) {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return nil, err
+	}
+	self, err := os.Executable()
+	if err != nil {
+		return nil, err
+	}
+	args := []string{
+		"-namespace", ns,
+		"-address", containerdAddress,
+		"-publish-binary", self,
+	}
+	args = append(args, cmdArgs...)
+	name := BinaryName(runtime)
+	if _, err := exec.LookPath(name); err != nil {
+		if eerr, ok := err.(*exec.Error); ok {
+			if eerr.Err == exec.ErrNotFound {
+				return nil, errors.Wrapf(os.ErrNotExist, "runtime %q binary not installed %q", runtime, name)
+			}
+		}
+	}
+	cmd := exec.Command(name, args...)
+	cmd.Dir = path
+	cmd.Env = append(os.Environ(), "GOMAXPROCS=2")
+	cmd.SysProcAttr = getSysProcAttr()
+	return cmd, nil
+}
+
+// BinaryName returns the shim binary name from the runtime name
+func BinaryName(runtime string) string {
+	parts := strings.Split(runtime, ".")
+	// TODO: add validation for runtime
+	return fmt.Sprintf(shimBinaryFormat, parts[len(parts)-2], parts[len(parts)-1])
+}
+
+// AbstractAddress returns an abstract socket address
+func AbstractAddress(ctx context.Context, id string) (string, error) {
+	ns, err := namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return "", err
+	}
+	return filepath.Join(string(filepath.Separator), "containerd-shim", ns, id, "shim.sock"), nil
+}
+
+// Connect to the provided address
+func Connect(address string, d func(string, time.Duration) (net.Conn, error)) (net.Conn, error) {
+	return d(address, 100*time.Second)
+}
+
+// AnonDialer returns a dialer for an abstract socket
+func AnonDialer(address string, timeout time.Duration) (net.Conn, error) {
+	address = strings.TrimPrefix(address, "unix://")
+	return net.DialTimeout("unix", "\x00"+address, timeout)
+}
+
+// NewSocket returns a new socket
+func NewSocket(address string) (*net.UnixListener, error) {
+	if len(address) > 106 {
+		return nil, errors.Errorf("%q: unix socket path too long (> 106)", address)
+	}
+	l, err := net.Listen("unix", "\x00"+address)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to listen to abstract unix socket %q", address)
+	}
+	return l.(*net.UnixListener), nil
+}
+
+// WritePidFile writes a pid file atomically
+func WritePidFile(path string, pid int) error {
+	path, err := filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
+	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
+	if err != nil {
+		return err
+	}
+	_, err = fmt.Fprintf(f, "%d", pid)
+	f.Close()
+	if err != nil {
+		return err
+	}
+	return os.Rename(tempPath, path)
+}
+
+// WriteAddress writes a address file atomically
+func WriteAddress(path, address string) error {
+	path, err := filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+	tempPath := filepath.Join(filepath.Dir(path), fmt.Sprintf(".%s", filepath.Base(path)))
+	f, err := os.OpenFile(tempPath, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_SYNC, 0666)
+	if err != nil {
+		return err
+	}
+	_, err = f.WriteString(address)
+	f.Close()
+	if err != nil {
+		return err
+	}
+	return os.Rename(tempPath, path)
+}

runtime/v2/shim/util_linux.go

@@ -0,0 +1,34 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"syscall"
+
+	"github.com/containerd/containerd/sys"
+)
+
+func getSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+// SetScore sets the oom score for a process
+func SetScore(pid int) error {
+	return sys.SetOOMScore(pid, sys.OOMScoreMaxKillable)
+}

runtime/v2/shim/util_unix.go

@@ -0,0 +1,34 @@
+// +build !linux,!windows
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"syscall"
+)
+
+func getSysProcAttr() *syscall.SysProcAttr {
+	return &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+// SetScore sets the oom score for a process
+func SetScore(pid int) error {
+	return nil
+}

runtime/v2/shim/util_windows.go

@@ -0,0 +1,30 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package shim
+
+import (
+	"syscall"
+)
+
+func getSysProcAttr() *syscall.SysProcAttr {
+	return nil
+}
+
+// SetScore sets the oom score for a process
+func SetScore(pid int) error {
+	return nil
+}

runtime/v2/task/doc.go

@@ -0,0 +1,17 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package task

runtime/v2/task/shim.proto

@@ -0,0 +1,184 @@
+syntax = "proto3";
+
+package containerd.task.v2;
+
+import "google/protobuf/any.proto";
+import "google/protobuf/empty.proto";
+import weak "gogoproto/gogo.proto";
+import "google/protobuf/timestamp.proto";
+import "github.com/containerd/containerd/api/types/mount.proto";
+import "github.com/containerd/containerd/api/types/task/task.proto";
+
+option go_package = "github.com/containerd/containerd/runtime/v2/task;task";
+
+// Shim service is launched for each container and is responsible for owning the IO
+// for the container and its additional processes.  The shim is also the parent of
+// each container and allows reattaching to the IO and receiving the exit status
+// for the container processes.
+service Task {
+	rpc State(StateRequest) returns (StateResponse);
+	rpc Create(CreateTaskRequest) returns (CreateTaskResponse);
+	rpc Start(StartRequest) returns (StartResponse);
+	rpc Delete(DeleteRequest) returns (DeleteResponse);
+	rpc Pids(PidsRequest) returns (PidsResponse);
+	rpc Pause(PauseRequest) returns (google.protobuf.Empty);
+	rpc Resume(ResumeRequest) returns (google.protobuf.Empty);
+	rpc Checkpoint(CheckpointTaskRequest) returns (google.protobuf.Empty);
+	rpc Kill(KillRequest) returns (google.protobuf.Empty);
+	rpc Exec(ExecProcessRequest) returns (google.protobuf.Empty);
+	rpc ResizePty(ResizePtyRequest) returns (google.protobuf.Empty);
+	rpc CloseIO(CloseIORequest) returns (google.protobuf.Empty);
+	rpc Update(UpdateTaskRequest) returns (google.protobuf.Empty);
+	rpc Wait(WaitRequest) returns (WaitResponse);
+	rpc Stats(StatsRequest) returns (StatsResponse);
+	rpc Connect(ConnectRequest) returns (ConnectResponse);
+	rpc Shutdown(ShutdownRequest) returns (google.protobuf.Empty);
+}
+
+message CreateTaskRequest {
+	string id = 1;
+	string bundle = 2;
+	repeated containerd.types.Mount rootfs = 3;
+	bool terminal = 4;
+	string stdin = 5;
+	string stdout = 6;
+	string stderr = 7;
+	string checkpoint = 8;
+	string parent_checkpoint = 9;
+	google.protobuf.Any options = 10;
+}
+
+message CreateTaskResponse {
+	uint32 pid = 1;
+}
+
+message DeleteRequest {
+	string id = 1;
+	string exec_id = 2;
+}
+
+message DeleteResponse {
+	uint32 pid = 1;
+	uint32 exit_status = 2;
+	google.protobuf.Timestamp exited_at = 3 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+}
+
+message ExecProcessRequest {
+	string id = 1;
+	string exec_id = 2;
+	bool terminal = 3;
+	string stdin = 4;
+	string stdout = 5;
+	string stderr = 6;
+	google.protobuf.Any spec = 7;
+}
+
+message ExecProcessResponse {
+}
+
+message ResizePtyRequest {
+	string id = 1;
+	string exec_id = 2;
+	uint32 width = 3;
+	uint32 height = 4;
+}
+
+message StateRequest {
+	string id = 1;
+	string exec_id = 2;
+}
+
+message StateResponse {
+	string id = 1;
+	string bundle = 2;
+	uint32 pid = 3;
+	containerd.v1.types.Status status = 4;
+	string stdin = 5;
+	string stdout = 6;
+	string stderr = 7;
+	bool terminal = 8;
+	uint32 exit_status = 9;
+	google.protobuf.Timestamp exited_at = 10 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+}
+
+message KillRequest {
+	string id = 1;
+	string exec_id = 2;
+	uint32 signal = 3;
+	bool all = 4;
+}
+
+message CloseIORequest {
+	string id = 1;
+	string exec_id = 2;
+	bool stdin = 3;
+}
+
+message PidsRequest {
+	string id = 1;
+}
+
+message PidsResponse {
+	repeated containerd.v1.types.ProcessInfo processes = 1;
+}
+
+message CheckpointTaskRequest {
+	string id = 1;
+	string path = 2;
+	google.protobuf.Any options = 3;
+}
+
+message UpdateTaskRequest {
+	string id = 1;
+	google.protobuf.Any resources = 2;
+}
+
+message StartRequest {
+	string id = 1;
+	string exec_id = 2;
+}
+
+message StartResponse {
+	uint32 pid = 1;
+}
+
+message WaitRequest {
+	string id = 1;
+	string exec_id = 2;
+}
+
+message WaitResponse {
+	uint32 exit_status = 1;
+	google.protobuf.Timestamp exited_at = 2 [(gogoproto.stdtime) = true, (gogoproto.nullable) = false];
+}
+
+message StatsRequest {
+	string id = 1;
+}
+
+message StatsResponse {
+	google.protobuf.Any stats = 1;
+}
+
+message ConnectRequest {
+	string id = 1;
+}
+
+message ConnectResponse {
+	uint32 shim_pid = 1;
+	uint32 task_pid = 2;
+	string version = 3;
+}
+
+message ShutdownRequest {
+	string id = 1;
+	bool now = 2;
+}
+
+message PauseRequest {
+	string id = 1;
+}
+
+message ResumeRequest {
+	string id = 1;
+}

services/tasks/local.go

@@ -42,6 +42,7 @@ import (
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/plugin"
 	"github.com/containerd/containerd/runtime"
+	"github.com/containerd/containerd/runtime/v2"
 	"github.com/containerd/containerd/services"
 	"github.com/containerd/typeurl"
 	ptypes "github.com/gogo/protobuf/types"
@@ -63,7 +64,9 @@ func init() {
 		ID:   services.TasksService,
 		Requires: []plugin.Type{
 			plugin.RuntimePlugin,
+			plugin.RuntimePluginV2,
 			plugin.MetadataPlugin,
+			plugin.TaskMonitorPlugin,
 		},
 		InitFn: initFunc,
 	})
@@ -75,11 +78,15 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) {
 		return nil, err
 	}
 
+	v2r, err := ic.Get(plugin.RuntimePluginV2)
+	if err != nil {
+		return nil, err
+	}
+
 	m, err := ic.Get(plugin.MetadataPlugin)
 	if err != nil {
 		return nil, err
 	}
-	cs := m.(*metadata.DB).ContentStore()
 	runtimes := make(map[string]runtime.PlatformRuntime)
 	for _, rr := range rt {
 		ri, err := rr.Instance()
@@ -94,12 +101,33 @@ func initFunc(ic *plugin.InitContext) (interface{}, error) {
 	if len(runtimes) == 0 {
 		return nil, errors.New("no runtimes available to create task service")
 	}
-	return &local{
+	monitor, err := ic.Get(plugin.TaskMonitorPlugin)
+	if err != nil {
+		if !errdefs.IsNotFound(err) {
+			return nil, err
+		}
+		monitor = runtime.NewNoopMonitor()
+	}
+
+	cs := m.(*metadata.DB).ContentStore()
+	l := &local{
 		runtimes:  runtimes,
 		db:        m.(*metadata.DB),
 		store:     cs,
 		publisher: ic.Events,
-	}, nil
+		monitor:   monitor.(runtime.TaskMonitor),
+		v2Runtime: v2r.(*v2.TaskManager),
+	}
+	for _, r := range runtimes {
+		tasks, err := r.Tasks(ic.Context, true)
+		if err != nil {
+			return nil, err
+		}
+		for _, t := range tasks {
+			l.monitor.Monitor(t)
+		}
+	}
+	return l, nil
 }
 
 type local struct {
@@ -107,6 +135,9 @@ type local struct {
 	db        *metadata.DB
 	store     content.Store
 	publisher events.Publisher
+
+	monitor   runtime.TaskMonitor
+	v2Runtime *v2.TaskManager
 }
 
 func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.CallOption) (*api.CreateTaskResponse, error) {
@@ -136,7 +167,6 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
 			return nil, err
 		}
 	}
-
 	container, err := l.getContainer(ctx, r.ContainerID)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
@@ -150,7 +180,9 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
 			Terminal: r.Terminal,
 		},
 		Checkpoint:     checkpointPath,
-		Options:    r.Options,
+		Runtime:        container.Runtime.Name,
+		RuntimeOptions: container.Runtime.Options,
+		TaskOptions:    r.Options,
 	}
 	for _, m := range r.Rootfs {
 		opts.Rootfs = append(opts.Rootfs, mount.Mount{
@@ -167,11 +199,14 @@ func (l *local) Create(ctx context.Context, r *api.CreateTaskRequest, _ ...grpc.
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
+	// TODO: fast path for getting pid on create
+	if err := l.monitor.Monitor(c); err != nil {
+		return nil, errors.Wrap(err, "monitor task")
+	}
 	state, err := c.State(ctx)
 	if err != nil {
 		log.G(ctx).Error(err)
 	}
-
 	return &api.CreateTaskResponse{
 		ContainerID: r.ContainerID,
 		Pid:         state.Pid,
@@ -206,13 +241,12 @@ func (l *local) Delete(ctx context.Context, r *api.DeleteTaskRequest, _ ...grpc.
 	if err != nil {
 		return nil, err
 	}
-	runtime, err := l.getRuntime(t.Info().Runtime)
-	if err != nil {
+	if err := l.monitor.Stop(t); err != nil {
 		return nil, err
 	}
-	exit, err := runtime.Delete(ctx, t)
+	exit, err := t.Delete(ctx)
 	if err != nil {
-		return nil, errdefs.ToGRPC(err)
+		return nil, err
 	}
 	return &api.DeleteResponse{
 		ExitStatus: exit.Status,
@@ -226,7 +260,11 @@ func (l *local) DeleteProcess(ctx context.Context, r *api.DeleteProcessRequest,
 	if err != nil {
 		return nil, err
 	}
-	exit, err := t.DeleteProcess(ctx, r.ExecID)
+	process, err := t.Process(ctx, r.ExecID)
+	if err != nil {
+		return nil, err
+	}
+	exit, err := process.Delete(ctx)
 	if err != nil {
 		return nil, errdefs.ToGRPC(err)
 	}
@@ -293,8 +331,8 @@ func (l *local) Get(ctx context.Context, r *api.GetRequest, _ ...grpc.CallOption
 
 func (l *local) List(ctx context.Context, r *api.ListTasksRequest, _ ...grpc.CallOption) (*api.ListTasksResponse, error) {
 	resp := &api.ListTasksResponse{}
-	for _, r := range l.runtimes {
-		tasks, err := r.Tasks(ctx)
+	for _, r := range l.allRuntimes() {
+		tasks, err := r.Tasks(ctx, false)
 		if err != nil {
 			return nil, errdefs.ToGRPC(err)
 		}
@@ -508,8 +546,8 @@ func (l *local) Metrics(ctx context.Context, r *api.MetricsRequest, _ ...grpc.Ca
 		return nil, err
 	}
 	var resp api.MetricsResponse
-	for _, r := range l.runtimes {
-		tasks, err := r.Tasks(ctx)
+	for _, r := range l.allRuntimes() {
+		tasks, err := r.Tasks(ctx, false)
 		if err != nil {
 			return nil, err
 		}
@@ -547,32 +585,26 @@ func getTasksMetrics(ctx context.Context, filter filters.Filter, tasks []runtime
 			case "id":
 				return t.ID(), true
 			case "namespace":
-				return t.Info().Namespace, true
+				return t.Namespace(), true
 			case "runtime":
-				return t.Info().Runtime, true
+				// return t.Info().Runtime, true
 			}
 			return "", false
 		})) {
 			continue
 		}
-
 		collected := time.Now()
-		metrics, err := tk.Metrics(ctx)
+		stats, err := tk.Stats(ctx)
 		if err != nil {
 			if !errdefs.IsNotFound(err) {
 				log.G(ctx).WithError(err).Errorf("collecting metrics for %s", tk.ID())
 			}
 			continue
 		}
-		data, err := typeurl.MarshalAny(metrics)
-		if err != nil {
-			log.G(ctx).WithError(err).Errorf("marshal metrics for %s", tk.ID())
-			continue
-		}
 		r.Metrics = append(r.Metrics, &types.Metric{
-			ID:        tk.ID(),
 			Timestamp: collected,
-			Data:      data,
+			ID:        tk.ID(),
+			Data:      stats,
 		})
 	}
 }
@@ -633,7 +665,16 @@ func (l *local) getTaskFromContainer(ctx context.Context, container *containers.
 func (l *local) getRuntime(name string) (runtime.PlatformRuntime, error) {
 	runtime, ok := l.runtimes[name]
 	if !ok {
-		return nil, status.Errorf(codes.NotFound, "unknown runtime %q", name)
+		// one runtime to rule them all
+		return l.v2Runtime, nil
 	}
 	return runtime, nil
 }
+
+func (l *local) allRuntimes() (o []runtime.PlatformRuntime) {
+	for _, r := range l.runtimes {
+		o = append(o, r)
+	}
+	o = append(o, l.v2Runtime)
+	return o
+}

sys/reaper.go

@@ -18,7 +18,9 @@
 
 package sys
 
-import "golang.org/x/sys/unix"
+import (
+	"golang.org/x/sys/unix"
+)
 
 // Exit is the wait4 information from an exited process
 type Exit struct {

sys/reaper_linux.go

@@ -0,0 +1,52 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package sys
+
+import (
+	"unsafe"
+
+	"golang.org/x/sys/unix"
+)
+
+// If arg2 is nonzero, set the "child subreaper" attribute of the
+// calling process; if arg2 is zero, unset the attribute.  When a
+// process is marked as a child subreaper, all of the children
+// that it creates, and their descendants, will be marked as
+// having a subreaper.  In effect, a subreaper fulfills the role
+// of init(1) for its descendant processes.  Upon termination of
+// a process that is orphaned (i.e., its immediate parent has
+// already terminated) and marked as having a subreaper, the
+// nearest still living ancestor subreaper will receive a SIGCHLD
+// signal and be able to wait(2) on the process to discover its
+// termination status.
+const setChildSubreaper = 36
+
+// SetSubreaper sets the value i as the subreaper setting for the calling process
+func SetSubreaper(i int) error {
+	return unix.Prctl(setChildSubreaper, uintptr(i), 0, 0, 0)
+}
+
+// GetSubreaper returns the subreaper setting for the calling process
+func GetSubreaper() (int, error) {
+	var i uintptr
+
+	if err := unix.Prctl(unix.PR_GET_CHILD_SUBREAPER, uintptr(unsafe.Pointer(&i)), 0, 0, 0); err != nil {
+		return -1, err
+	}
+
+	return int(i), nil
+}

task_opts.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/mount"
-	"github.com/containerd/containerd/runtime/linux/runctypes"
 )
 
 // NewTaskOpts allows the caller to set options on a new task
@@ -36,14 +35,6 @@ func WithRootFS(mounts []mount.Mount) NewTaskOpts {
 	}
 }
 
-// WithExit causes the task to exit after a successful checkpoint
-func WithExit(r *CheckpointTaskInfo) error {
-	r.Options = &runctypes.CheckpointOptions{
-		Exit: true,
-	}
-	return nil
-}
-
 // WithCheckpointName sets the image name for the checkpoint
 func WithCheckpointName(name string) CheckpointTaskOpts {
 	return func(r *CheckpointTaskInfo) error {

windows/process.go

@@ -231,3 +231,21 @@ func (p *process) Wait(ctx context.Context) (*runtime.Exit, error) {
 		Timestamp: ea,
 	}, nil
 }
+
+func (p *process) Delete(ctx context.Context) (*runtime.Exit, error) {
+	ec, ea, err := p.ExitCode()
+	if err != nil {
+		return nil, err
+	}
+	// If we never started the process close the pipes
+	if p.Status() == runtime.CreatedStatus {
+		p.io.Close()
+		ea = time.Now()
+	}
+	p.task.removeProcess(p.id)
+	return &runtime.Exit{
+		Pid:       p.pid,
+		Status:    ec,
+		Timestamp: ea,
+	}, nil
+}

windows/runtime.go

@@ -79,8 +79,6 @@ func New(ic *plugin.InitContext) (interface{}, error) {
 
 		events:    make(chan interface{}, 4096),
 		publisher: ic.Events,
-		// TODO(mlaventure): windows needs a stat monitor
-		monitor: nil,
 		tasks:     runtime.NewTaskList(),
 	}
 
@@ -100,7 +98,6 @@ type windowsRuntime struct {
 	publisher events.Publisher
 	events    chan interface{}
 
-	monitor runtime.TaskMonitor
 	tasks *runtime.TaskList
 }
 
@@ -121,8 +118,8 @@ func (r *windowsRuntime) Create(ctx context.Context, id string, opts runtime.Cre
 	spec := s.(*runtimespec.Spec)
 
 	var createOpts *hcsshimtypes.CreateOptions
-	if opts.Options != nil {
-		o, err := typeurl.UnmarshalAny(opts.Options)
+	if opts.TaskOptions != nil {
+		o, err := typeurl.UnmarshalAny(opts.TaskOptions)
 		if err != nil {
 			return nil, err
 		}
@@ -152,74 +149,8 @@ func (r *windowsRuntime) Get(ctx context.Context, id string) (runtime.Task, erro
 	return r.tasks.Get(ctx, id)
 }
 
-func (r *windowsRuntime) Tasks(ctx context.Context) ([]runtime.Task, error) {
-	return r.tasks.GetAll(ctx)
-}
-
-func (r *windowsRuntime) Delete(ctx context.Context, t runtime.Task) (*runtime.Exit, error) {
-	wt, ok := t.(*task)
-	if !ok {
-		return nil, errors.Wrap(errdefs.ErrInvalidArgument, "no a windows task")
-	}
-
-	// TODO(mlaventure): stop monitor on this task
-
-	var (
-		err      error
-		state, _ = wt.State(ctx)
-	)
-	switch state.Status {
-	case runtime.StoppedStatus:
-		fallthrough
-	case runtime.CreatedStatus:
-		// if it's stopped or in created state, we need to shutdown the
-		// container before removing it
-		if err = wt.stop(ctx); err != nil {
-			return nil, err
-		}
-	default:
-		return nil, errors.Wrap(errdefs.ErrFailedPrecondition,
-			"cannot delete a non-stopped task")
-	}
-
-	var rtExit *runtime.Exit
-	if p := wt.getProcess(t.ID()); p != nil {
-		ec, ea, err := p.ExitCode()
-		if err != nil {
-			return nil, err
-		}
-		rtExit = &runtime.Exit{
-			Pid:       wt.pid,
-			Status:    ec,
-			Timestamp: ea,
-		}
-	} else {
-		rtExit = &runtime.Exit{
-			Pid:       wt.pid,
-			Status:    255,
-			Timestamp: time.Now().UTC(),
-		}
-	}
-
-	wt.cleanup()
-	r.tasks.Delete(ctx, t.ID())
-
-	r.publisher.Publish(ctx,
-		runtime.TaskDeleteEventTopic,
-		&eventstypes.TaskDelete{
-			ContainerID: wt.id,
-			Pid:         wt.pid,
-			ExitStatus:  rtExit.Status,
-			ExitedAt:    rtExit.Timestamp,
-		})
-
-	if err := mount.UnmountAll(wt.rootfs[0].Source, 0); err != nil {
-		log.G(ctx).WithError(err).WithField("path", wt.rootfs[0].Source).
-			Warn("failed to unmount rootfs on failure")
-	}
-
-	// We were never started, return failure
-	return rtExit, nil
+func (r *windowsRuntime) Tasks(ctx context.Context, all bool) ([]runtime.Task, error) {
+	return r.tasks.GetAll(ctx, all)
 }
 
 func (r *windowsRuntime) newTask(ctx context.Context, namespace, id string, rootfs []mount.Mount, spec *runtimespec.Spec, io runtime.IO, createOpts *hcsshimtypes.CreateOptions) (*task, error) {
@@ -298,6 +229,7 @@ func (r *windowsRuntime) newTask(ctx context.Context, namespace, id string, root
 		pidPool:           r.pidPool,
 		hcsContainer:      ctr,
 		terminateDuration: createOpts.TerminateDuration,
+		tasks:             r.tasks,
 	}
 	// Create the new process but don't start it
 	pconf := newWindowsProcessConfig(t.spec.Process, t.io)

windows/task.go

@@ -27,6 +27,7 @@ import (
 	eventstypes "github.com/containerd/containerd/api/events"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/log"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/runtime"
 	"github.com/containerd/containerd/windows/hcsshimtypes"
@@ -55,12 +56,17 @@ type task struct {
 	pidPool           *pidPool
 	hcsContainer      hcsshim.Container
 	terminateDuration time.Duration
+	tasks             *runtime.TaskList
 }
 
 func (t *task) ID() string {
 	return t.id
 }
 
+func (t *task) Namespace() string {
+	return t.namespace
+}
+
 func (t *task) State(ctx context.Context) (runtime.State, error) {
 	var (
 		status     runtime.Status
@@ -271,33 +277,6 @@ func (t *task) Checkpoint(_ context.Context, _ string, _ *types.Any) error {
 	return errors.Wrap(errdefs.ErrUnavailable, "not supported")
 }
 
-func (t *task) DeleteProcess(ctx context.Context, id string) (*runtime.Exit, error) {
-	if id == t.id {
-		return nil, errors.Wrapf(errdefs.ErrInvalidArgument,
-			"cannot delete init process")
-	}
-	if p := t.getProcess(id); p != nil {
-		ec, ea, err := p.ExitCode()
-		if err != nil {
-			return nil, err
-		}
-
-		// If we never started the process close the pipes
-		if p.Status() == runtime.CreatedStatus {
-			p.io.Close()
-			ea = time.Now()
-		}
-
-		t.removeProcess(id)
-		return &runtime.Exit{
-			Pid:       p.pid,
-			Status:    ec,
-			Timestamp: ea,
-		}, nil
-	}
-	return nil, errors.Wrapf(errdefs.ErrNotFound, "no such process %s", id)
-}
-
 func (t *task) Update(ctx context.Context, resources *types.Any) error {
 	return errors.Wrap(errdefs.ErrUnavailable, "not supported")
 }
@@ -311,7 +290,7 @@ func (t *task) Process(ctx context.Context, id string) (p runtime.Process, err e
 	return p, err
 }
 
-func (t *task) Metrics(ctx context.Context) (interface{}, error) {
+func (t *task) Stats(ctx context.Context) (*types.Any, error) {
 	return nil, errors.Wrap(errdefs.ErrUnavailable, "not supported")
 }
 
@@ -323,6 +302,64 @@ func (t *task) Wait(ctx context.Context) (*runtime.Exit, error) {
 	return p.Wait(ctx)
 }
 
+func (t *task) Delete(ctx context.Context) (*runtime.Exit, error) {
+	var (
+		err      error
+		state, _ = t.State(ctx)
+	)
+	switch state.Status {
+	case runtime.StoppedStatus:
+		fallthrough
+	case runtime.CreatedStatus:
+		// if it's stopped or in created state, we need to shutdown the
+		// container before removing it
+		if err = t.stop(ctx); err != nil {
+			return nil, err
+		}
+	default:
+		return nil, errors.Wrap(errdefs.ErrFailedPrecondition,
+			"cannot delete a non-stopped task")
+	}
+
+	var rtExit *runtime.Exit
+	if p := t.getProcess(t.ID()); p != nil {
+		ec, ea, err := p.ExitCode()
+		if err != nil {
+			return nil, err
+		}
+		rtExit = &runtime.Exit{
+			Pid:       t.pid,
+			Status:    ec,
+			Timestamp: ea,
+		}
+	} else {
+		rtExit = &runtime.Exit{
+			Pid:       t.pid,
+			Status:    255,
+			Timestamp: time.Now().UTC(),
+		}
+	}
+
+	t.cleanup()
+	t.tasks.Delete(ctx, t.ID())
+
+	t.publisher.Publish(ctx,
+		runtime.TaskDeleteEventTopic,
+		&eventstypes.TaskDelete{
+			ContainerID: t.id,
+			Pid:         t.pid,
+			ExitStatus:  rtExit.Status,
+			ExitedAt:    rtExit.Timestamp,
+		})
+
+	if err := mount.UnmountAll(t.rootfs[0].Source, 0); err != nil {
+		log.G(ctx).WithError(err).WithField("path", t.rootfs[0].Source).
+			Warn("failed to unmount rootfs on failure")
+	}
+	// We were never started, return failure
+	return rtExit, nil
+}
+
 func (t *task) newProcess(ctx context.Context, id string, conf *hcsshim.ProcessConfig, pset *pipeSet) (*process, error) {
 	var (
 		err error