Merge pull request #618 from Random-Liu/add-generic-restart-test

Add generic restart test

.travis.yml

@@ -53,7 +53,7 @@ jobs:
         - cat /tmp/test-cri/cri-containerd.log
         - cat /tmp/test-cri/containerd.log
       go: "1.9.x"
-      script:
+    - script:
         - make install.deps
         - make test
         - make test-integration

hack/install-deps.sh

@@ -133,7 +133,8 @@ fi
 # Install containerd
 checkout_repo ${CONTAINERD_PKG} ${CONTAINERD_VERSION} ${CONTAINERD_REPO}
 cd ${GOPATH}/src/${CONTAINERD_PKG}
-make BUILDTAGS="${BUILDTAGS}"
+# Build no_cri version and run standalone cri-containerd.
+make BUILDTAGS="${BUILDTAGS} no_cri"
 # containerd make install requires `go` to work. Explicitly
 # set PATH to make sure it can find `go` even with `sudo`.
 ${sudo} sh -c "PATH=${PATH} make install -e DESTDIR=${CONTAINERD_DIR}"

integration/restart_test.go

@@ -17,8 +17,6 @@ limitations under the License.
 package integration
 
 import (
-	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -26,47 +24,61 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/context"
-	"golang.org/x/sys/unix"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
 // Restart test must run sequentially.
-// NOTE(random-liu): Current restart test only support standalone cri-containerd mode.
 
-func TestSandboxAcrossCRIContainerdRestart(t *testing.T) {
-	if !*standaloneCRIContainerd {
-		t.Skip("Skip because cri-containerd does not run in standalone mode")
-	}
-	ctx := context.Background()
-	sandboxNS := "sandbox-restart-cri-containerd"
-	sandboxes := []struct {
+func TestContainerdRestart(t *testing.T) {
+	type container struct {
 		name  string
 		id    string
-		stateBeforeExit runtime.PodSandboxState
-		actionAfterExit string
-		expectedState   runtime.PodSandboxState
-	}{
+		state runtime.ContainerState
+	}
+	type sandbox struct {
+		name       string
+		id         string
+		state      runtime.PodSandboxState
+		containers []container
+	}
+	ctx := context.Background()
+	sandboxNS := "restart-containerd"
+	sandboxes := []sandbox{
 		{
-			name:            "task-always-ready",
-			stateBeforeExit: runtime.PodSandboxState_SANDBOX_READY,
-			expectedState:   runtime.PodSandboxState_SANDBOX_READY,
+			name:  "ready-sandbox",
+			state: runtime.PodSandboxState_SANDBOX_READY,
+			containers: []container{
+				{
+					name:  "created-container",
+					state: runtime.ContainerState_CONTAINER_CREATED,
 				},
 				{
-			name:            "task-always-not-ready",
-			stateBeforeExit: runtime.PodSandboxState_SANDBOX_NOTREADY,
-			expectedState:   runtime.PodSandboxState_SANDBOX_NOTREADY,
+					name:  "running-container",
+					state: runtime.ContainerState_CONTAINER_RUNNING,
 				},
 				{
-			name:            "task-exit-before-restart",
-			stateBeforeExit: runtime.PodSandboxState_SANDBOX_READY,
-			actionAfterExit: "kill",
-			expectedState:   runtime.PodSandboxState_SANDBOX_NOTREADY,
+					name:  "exited-container",
+					state: runtime.ContainerState_CONTAINER_EXITED,
+				},
+			},
 		},
 		{
-			name:            "task-deleted-before-restart",
-			stateBeforeExit: runtime.PodSandboxState_SANDBOX_READY,
-			actionAfterExit: "delete",
-			expectedState:   runtime.PodSandboxState_SANDBOX_NOTREADY,
+			name:  "notready-sandbox",
+			state: runtime.PodSandboxState_SANDBOX_NOTREADY,
+			containers: []container{
+				{
+					name:  "created-container",
+					state: runtime.ContainerState_CONTAINER_CREATED,
+				},
+				{
+					name:  "running-container",
+					state: runtime.ContainerState_CONTAINER_RUNNING,
+				},
+				{
+					name:  "exited-container",
+					state: runtime.ContainerState_CONTAINER_EXITED,
+				},
+			},
 		},
 	}
 	t.Logf("Make sure no sandbox is running before test")
@@ -74,134 +86,97 @@ func TestSandboxAcrossCRIContainerdRestart(t *testing.T) {
 	require.NoError(t, err)
 	require.Empty(t, existingSandboxes)
 
-	t.Logf("Start test sandboxes")
+	t.Logf("Start test sandboxes and containers")
 	for i := range sandboxes {
 		s := &sandboxes[i]
-		cfg := PodSandboxConfig(s.name, sandboxNS)
-		sb, err := runtimeService.RunPodSandbox(cfg)
+		sbCfg := PodSandboxConfig(s.name, sandboxNS)
+		sid, err := runtimeService.RunPodSandbox(sbCfg)
 		require.NoError(t, err)
 		defer func() {
 			// Make sure the sandbox is cleaned up in any case.
-			runtimeService.StopPodSandbox(sb)
-			runtimeService.RemovePodSandbox(sb)
+			runtimeService.StopPodSandbox(sid)
+			runtimeService.RemovePodSandbox(sid)
 		}()
-		s.id = sb
-		if s.stateBeforeExit == runtime.PodSandboxState_SANDBOX_NOTREADY {
-			require.NoError(t, runtimeService.StopPodSandbox(sb))
+		s.id = sid
+		for j := range s.containers {
+			c := &s.containers[j]
+			cfg := ContainerConfig(c.name, pauseImage,
+				// Set pid namespace as per container, so that container won't die
+				// when sandbox container is killed.
+				WithPidNamespace(runtime.NamespaceMode_CONTAINER),
+			)
+			cid, err := runtimeService.CreateContainer(sid, cfg, sbCfg)
+			require.NoError(t, err)
+			// Reply on sandbox cleanup.
+			c.id = cid
+			switch c.state {
+			case runtime.ContainerState_CONTAINER_CREATED:
+			case runtime.ContainerState_CONTAINER_RUNNING:
+				require.NoError(t, runtimeService.StartContainer(cid))
+			case runtime.ContainerState_CONTAINER_EXITED:
+				require.NoError(t, runtimeService.StartContainer(cid))
+				require.NoError(t, runtimeService.StopContainer(cid, 10))
 			}
 		}
-
-	t.Logf("Kill cri-containerd")
-	require.NoError(t, KillProcess("cri-containerd"))
-	defer func() {
-		assert.NoError(t, Eventually(func() (bool, error) {
-			return ConnectDaemons() == nil, nil
-		}, time.Second, 30*time.Second), "make sure cri-containerd is running before test finish")
-	}()
-
-	t.Logf("Change sandbox state, must finish before cri-containerd is restarted")
-	for _, s := range sandboxes {
-		if s.actionAfterExit == "" {
-			continue
-		}
-		cntr, err := containerdClient.LoadContainer(ctx, s.id)
+		if s.state == runtime.PodSandboxState_SANDBOX_NOTREADY {
+			cntr, err := containerdClient.LoadContainer(ctx, sid)
 			require.NoError(t, err)
 			task, err := cntr.Task(ctx, nil)
 			require.NoError(t, err)
-		switch s.actionAfterExit {
-		case "kill":
-			require.NoError(t, task.Kill(ctx, unix.SIGKILL, containerd.WithKillAll))
-		case "delete":
-			_, err := task.Delete(ctx, containerd.WithProcessKill)
+			_, err = task.Delete(ctx, containerd.WithProcessKill)
 			require.NoError(t, err)
 		}
 	}
 
-	t.Logf("Wait until cri-containerd is restarted")
+	t.Logf("Kill containerd")
+	require.NoError(t, KillProcess("containerd"))
+	defer func() {
+		assert.NoError(t, Eventually(func() (bool, error) {
+			return ConnectDaemons() == nil, nil
+		}, time.Second, 30*time.Second), "make sure containerd is running before test finish")
+	}()
+
+	t.Logf("Wait until containerd is killed")
+	require.NoError(t, Eventually(func() (bool, error) {
+		pid, err := PidOf("containerd")
+		if err != nil {
+			return false, err
+		}
+		return pid == 0, nil
+	}, time.Second, 30*time.Second), "wait for containerd to be killed")
+
+	t.Logf("Wait until containerd is restarted")
 	require.NoError(t, Eventually(func() (bool, error) {
 		return ConnectDaemons() == nil, nil
-	}, time.Second, 30*time.Second), "wait for cri-containerd to be restarted")
+	}, time.Second, 30*time.Second), "wait for containerd to be restarted")
 
-	t.Logf("Check sandbox state after restart")
+	t.Logf("Check sandbox and container state after restart")
 	loadedSandboxes, err := runtimeService.ListPodSandbox(&runtime.PodSandboxFilter{})
 	require.NoError(t, err)
 	assert.Len(t, loadedSandboxes, len(sandboxes))
+	loadedContainers, err := runtimeService.ListContainers(&runtime.ContainerFilter{})
+	require.NoError(t, err)
+	assert.Len(t, loadedContainers, len(sandboxes)*3)
 	for _, s := range sandboxes {
 		for _, loaded := range loadedSandboxes {
 			if s.id == loaded.Id {
-				assert.Equal(t, s.expectedState, loaded.State)
+				assert.Equal(t, s.state, loaded.State)
 				break
 			}
 		}
+		for _, c := range s.containers {
+			for _, loaded := range loadedContainers {
+				if c.id == loaded.Id {
+					assert.Equal(t, c.state, loaded.State)
+					break
+				}
+			}
+		}
 	}
 
 	t.Logf("Should be able to stop and remove sandbox after restart")
 	for _, s := range sandboxes {
-		// Properly stop the sandbox if it's ready before restart.
-		if s.stateBeforeExit == runtime.PodSandboxState_SANDBOX_READY {
 		assert.NoError(t, runtimeService.StopPodSandbox(s.id))
-		}
 		assert.NoError(t, runtimeService.RemovePodSandbox(s.id))
 	}
 }
-
-// TestSandboxDeletionAcrossCRIContainerdRestart tests the case that sandbox container
-// is deleted from containerd during cri-containerd is down. This should not happen.
-// However, if this really happens, cri-containerd should not load such sandbox and
-// should do best effort cleanup of the sandbox root directory. Note that in this case,
-// cri-containerd loses the network namespace of the sandbox, so it won't be able to
-// teardown the network properly.
-// This test uses host network sandbox to avoid resource leakage.
-func TestSandboxDeletionAcrossCRIContainerdRestart(t *testing.T) {
-	if !*standaloneCRIContainerd {
-		t.Skip("Skip because cri-containerd does not run in standalone mode")
-	}
-	ctx := context.Background()
-	sandboxNS := "sandbox-delete-restart-cri-containerd"
-	t.Logf("Make sure no sandbox is running before test")
-	existingSandboxes, err := runtimeService.ListPodSandbox(&runtime.PodSandboxFilter{})
-	require.NoError(t, err)
-	require.Empty(t, existingSandboxes)
-
-	t.Logf("Start test sandboxes")
-	cfg := PodSandboxConfig("sandbox", sandboxNS, WithHostNetwork)
-	sb, err := runtimeService.RunPodSandbox(cfg)
-	require.NoError(t, err)
-	defer func() {
-		// Make sure the sandbox is cleaned up in any case.
-		runtimeService.StopPodSandbox(sb)
-		runtimeService.RemovePodSandbox(sb)
-	}()
-
-	t.Logf("Kill cri-containerd")
-	require.NoError(t, KillProcess("cri-containerd"))
-	defer func() {
-		assert.NoError(t, Eventually(func() (bool, error) {
-			return ConnectDaemons() == nil, nil
-		}, time.Second, 30*time.Second), "make sure cri-containerd is running before test finish")
-	}()
-
-	t.Logf("Delete sandbox container from containerd")
-	cntr, err := containerdClient.LoadContainer(ctx, sb)
-	require.NoError(t, err)
-	task, err := cntr.Task(ctx, nil)
-	require.NoError(t, err)
-	_, err = task.Delete(ctx, containerd.WithProcessKill)
-	require.NoError(t, err)
-	require.NoError(t, cntr.Delete(ctx, containerd.WithSnapshotCleanup))
-
-	t.Logf("Wait until cri-containerd is restarted")
-	require.NoError(t, Eventually(func() (bool, error) {
-		return ConnectDaemons() == nil, nil
-	}, time.Second, 30*time.Second), "wait for cri-containerd to be restarted")
-
-	t.Logf("Check sandbox state after restart")
-	loadedSandboxes, err := runtimeService.ListPodSandbox(&runtime.PodSandboxFilter{})
-	require.NoError(t, err)
-	assert.Empty(t, loadedSandboxes)
-
-	t.Logf("Make sure sandbox root is removed")
-	sandboxRoot := filepath.Join(*criContainerdRoot, "sandboxes", sb)
-	_, err = os.Stat(sandboxRoot)
-	assert.True(t, os.IsNotExist(err))
-}

integration/test_utils.go

@@ -21,6 +21,8 @@ import (
 	"flag"
 	"fmt"
 	"os/exec"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/containerd/containerd"
@@ -161,6 +163,23 @@ func WithCommand(c string, args ...string) ContainerOpts {
 	}
 }
 
+// Add pid namespace mode.
+func WithPidNamespace(mode runtime.NamespaceMode) ContainerOpts {
+	return func(cf *runtime.ContainerConfig) {
+		if cf.Linux == nil {
+			cf.Linux = &runtime.LinuxContainerConfig{}
+		}
+		if cf.Linux.SecurityContext == nil {
+			cf.Linux.SecurityContext = &runtime.LinuxContainerSecurityContext{}
+		}
+		if cf.Linux.SecurityContext.NamespaceOptions == nil {
+			cf.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{}
+		}
+		cf.Linux.SecurityContext.NamespaceOptions.Pid = mode
+	}
+
+}
+
 // ContainerConfig creates a container config given a name and image name
 // and additional container config options
 func ContainerConfig(name, image string, opts ...ContainerOpts) *runtime.ContainerConfig {
@@ -212,3 +231,16 @@ func KillProcess(name string) error {
 	}
 	return nil
 }
+
+// PidOf returns pid of a process by name.
+func PidOf(name string) (int, error) {
+	b, err := exec.Command("pidof", name).CombinedOutput()
+	output := strings.TrimSpace(string(b))
+	if err != nil {
+		if len(output) != 0 {
+			return 0, fmt.Errorf("failed to run pidof %q - error: %v, output: %q", name, err, output)
+		}
+		return 0, nil
+	}
+	return strconv.Atoi(output)
+}

vendor.conf

@@ -4,7 +4,7 @@ github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd
 github.com/BurntSushi/toml a368813c5e648fee92e5f6c30e3944ff9d5e8895
 github.com/containerd/cgroups c0710c92e8b3a44681d1321dcfd1360fc5c6c089
 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e
-github.com/containerd/containerd 129167132c5e0dbd1b031badae201a432d1bd681
+github.com/containerd/containerd 25c403415aa99d0f3a609043429f3d24c8b70c0c
 github.com/containerd/continuity d8fb8589b0e8e85b8c8bbaa8840226d0dfeb7371
 github.com/containerd/fifo fbfb6a11ec671efbe94ad1c12c2e98773f19e1e6
 github.com/containerd/go-runc 4f6e87ae043f859a38255247b49c9abc262d002f

vendor/github.com/containerd/containerd/api/events/container.pb.go

@@ -158,9 +158,7 @@ func (m *ContainerCreate_Runtime) Field(fieldpath []string) (string, bool) {
 			return "", false
 		}
 
-		adaptor, ok := decoded.(interface {
-			Field([]string) (string, bool)
-		})
+		adaptor, ok := decoded.(interface{ Field([]string) (string, bool) })
 		if !ok {
 			return "", false
 		}

vendor/github.com/containerd/containerd/api/services/events/v1/events.pb.go

@@ -115,9 +115,7 @@ func (m *Envelope) Field(fieldpath []string) (string, bool) {
 			return "", false
 		}
 
-		adaptor, ok := decoded.(interface {
-			Field([]string) (string, bool)
-		})
+		adaptor, ok := decoded.(interface{ Field([]string) (string, bool) })
 		if !ok {
 			return "", false
 		}

vendor/github.com/containerd/containerd/cio/io_unix.go

@@ -114,17 +114,24 @@ func openFifos(ctx context.Context, fifos *FIFOSet) (pipes, error) {
 		if f.Stdin, err = fifo.OpenFifo(ctx, fifos.Stdin, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil {
 			return f, errors.Wrapf(err, "failed to open stdin fifo")
 		}
+		defer func() {
+			if err != nil && f.Stdin != nil {
+				f.Stdin.Close()
+			}
+		}()
 	}
 	if fifos.Stdout != "" {
 		if f.Stdout, err = fifo.OpenFifo(ctx, fifos.Stdout, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil {
-			f.Stdin.Close()
 			return f, errors.Wrapf(err, "failed to open stdout fifo")
 		}
+		defer func() {
+			if err != nil && f.Stdout != nil {
+				f.Stdout.Close()
+			}
+		}()
 	}
 	if fifos.Stderr != "" {
 		if f.Stderr, err = fifo.OpenFifo(ctx, fifos.Stderr, syscall.O_RDONLY|syscall.O_CREAT|syscall.O_NONBLOCK, 0700); err != nil {
-			f.Stdin.Close()
-			f.Stdout.Close()
 			return f, errors.Wrapf(err, "failed to open stderr fifo")
 		}
 	}

vendor/github.com/containerd/containerd/client.go

@@ -93,11 +93,22 @@ func New(address string, opts ...ClientOpt) (*Client, error) {
 			grpc.WithStreamInterceptor(stream),
 		)
 	}
+	connector := func() (*grpc.ClientConn, error) {
 		conn, err := grpc.Dial(dialer.DialAddress(address), gopts...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to dial %q", address)
 		}
-	return NewWithConn(conn, opts...)
+		return conn, nil
+	}
+	conn, err := connector()
+	if err != nil {
+		return nil, err
+	}
+	return &Client{
+		conn:      conn,
+		connector: connector,
+		runtime:   fmt.Sprintf("%s.%s", plugin.RuntimePlugin, runtime.GOOS),
+	}, nil
 }
 
 // NewWithConn returns a new containerd client that is connected to the containerd
@@ -114,6 +125,21 @@ func NewWithConn(conn *grpc.ClientConn, opts ...ClientOpt) (*Client, error) {
 type Client struct {
 	conn      *grpc.ClientConn
 	runtime   string
+	connector func() (*grpc.ClientConn, error)
+}
+
+// Reconnect re-establishes the GRPC connection to the containerd daemon
+func (c *Client) Reconnect() error {
+	if c.connector == nil {
+		return errors.New("unable to reconnect to containerd, no connector available")
+	}
+	c.conn.Close()
+	conn, err := c.connector()
+	if err != nil {
+		return err
+	}
+	c.conn = conn
+	return nil
 }
 
 // IsServing returns true if the client can successfully connect to the

vendor/github.com/containerd/containerd/cmd/containerd/command/main.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/server"
 	"github.com/containerd/containerd/sys"
 	"github.com/containerd/containerd/version"
@@ -111,6 +112,14 @@ func App() *cli.App {
 		if err := applyFlags(context, config); err != nil {
 			return err
 		}
+		// cleanup temp mounts
+		if err := mount.SetTempMountLocation(filepath.Join(config.Root, "tmpmounts")); err != nil {
+			return errors.Wrap(err, "creating temp mount location")
+		}
+		// unmount all temp mounts on boot for the server
+		if err := mount.CleanupTempMounts(0); err != nil {
+			return errors.Wrap(err, "unmounting temp mounts")
+		}
 		address := config.GRPC.Address
 		if address == "" {
 			return errors.New("grpc address cannot be empty")

vendor/github.com/containerd/containerd/linux/runtime.go

@@ -397,10 +397,6 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) {
 		ctx = namespaces.WithNamespace(ctx, ns)
 		pid, _ := runc.ReadPidFile(filepath.Join(bundle.path, proc.InitPidFile))
 		s, err := bundle.NewShimClient(ctx, ns, ShimConnect(func() {
-			log.G(ctx).WithError(err).WithFields(logrus.Fields{
-				"id":        id,
-				"namespace": ns,
-			}).Error("connecting to shim")
 			err := r.cleanupAfterDeadShim(ctx, bundle, ns, id, pid)
 			if err != nil {
 				log.G(ctx).WithError(err).WithField("bundle", bundle.path).

vendor/github.com/containerd/containerd/mount/mount.go

@@ -16,15 +16,6 @@
 
 package mount
 
-import (
-	"context"
-	"io/ioutil"
-	"os"
-
-	"github.com/containerd/containerd/log"
-	"github.com/pkg/errors"
-)
-
 // Mount is the lingua franca of containerd. A mount represents a
 // serialized mount syscall. Components either emit or consume mounts.
 type Mount struct {
@@ -47,42 +38,3 @@ func All(mounts []Mount, target string) error {
 	}
 	return nil
 }
-
-// WithTempMount mounts the provided mounts to a temp dir, and pass the temp dir to f.
-// The mounts are valid during the call to the f.
-// Finally we will unmount and remove the temp dir regardless of the result of f.
-func WithTempMount(ctx context.Context, mounts []Mount, f func(root string) error) (err error) {
-	root, uerr := ioutil.TempDir("", "containerd-WithTempMount")
-	if uerr != nil {
-		return errors.Wrapf(uerr, "failed to create temp dir")
-	}
-	// We use Remove here instead of RemoveAll.
-	// The RemoveAll will delete the temp dir and all children it contains.
-	// When the Unmount fails, RemoveAll will incorrectly delete data from
-	// the mounted dir. However, if we use Remove, even though we won't
-	// successfully delete the temp dir and it may leak, we won't loss data
-	// from the mounted dir.
-	// For details, please refer to #1868 #1785.
-	defer func() {
-		if uerr = os.Remove(root); uerr != nil {
-			log.G(ctx).WithError(uerr).WithField("dir", root).Errorf("failed to remove mount temp dir")
-		}
-	}()
-
-	// We should do defer first, if not we will not do Unmount when only a part of Mounts are failed.
-	defer func() {
-		if uerr = UnmountAll(root, 0); uerr != nil {
-			uerr = errors.Wrapf(uerr, "failed to unmount %s", root)
-			if err == nil {
-				err = uerr
-			} else {
-				err = errors.Wrap(err, uerr.Error())
-			}
-		}
-	}()
-	if uerr = All(mounts, root); uerr != nil {
-		return errors.Wrapf(uerr, "failed to mount %s", root)
-	}
-
-	return errors.Wrapf(f(root), "mount callback failed on %s", root)
-}

vendor/github.com/containerd/containerd/mount/temp.go

@@ -0,0 +1,50 @@
+package mount
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+
+	"github.com/containerd/containerd/log"
+	"github.com/pkg/errors"
+)
+
+var tempMountLocation = os.TempDir()
+
+// WithTempMount mounts the provided mounts to a temp dir, and pass the temp dir to f.
+// The mounts are valid during the call to the f.
+// Finally we will unmount and remove the temp dir regardless of the result of f.
+func WithTempMount(ctx context.Context, mounts []Mount, f func(root string) error) (err error) {
+	root, uerr := ioutil.TempDir(tempMountLocation, "containerd-mount")
+	if uerr != nil {
+		return errors.Wrapf(uerr, "failed to create temp dir")
+	}
+	// We use Remove here instead of RemoveAll.
+	// The RemoveAll will delete the temp dir and all children it contains.
+	// When the Unmount fails, RemoveAll will incorrectly delete data from
+	// the mounted dir. However, if we use Remove, even though we won't
+	// successfully delete the temp dir and it may leak, we won't loss data
+	// from the mounted dir.
+	// For details, please refer to #1868 #1785.
+	defer func() {
+		if uerr = os.Remove(root); uerr != nil {
+			log.G(ctx).WithError(uerr).WithField("dir", root).Errorf("failed to remove mount temp dir")
+		}
+	}()
+
+	// We should do defer first, if not we will not do Unmount when only a part of Mounts are failed.
+	defer func() {
+		if uerr = UnmountAll(root, 0); uerr != nil {
+			uerr = errors.Wrapf(uerr, "failed to unmount %s", root)
+			if err == nil {
+				err = uerr
+			} else {
+				err = errors.Wrap(err, uerr.Error())
+			}
+		}
+	}()
+	if uerr = All(mounts, root); uerr != nil {
+		return errors.Wrapf(uerr, "failed to mount %s", root)
+	}
+	return errors.Wrapf(f(root), "mount callback failed on %s", root)
+}

vendor/github.com/containerd/containerd/mount/temp_unix.go

@@ -0,0 +1,47 @@
+// +build !windows
+
+package mount
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// SetTempMountLocation sets the temporary mount location
+func SetTempMountLocation(root string) error {
+	root, err := filepath.Abs(root)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(root, 0700); err != nil {
+		return err
+	}
+	tempMountLocation = root
+	return nil
+}
+
+// CleanupTempMounts all temp mounts and remove the directories
+func CleanupTempMounts(flags int) error {
+	mounts, err := Self()
+	if err != nil {
+		return err
+	}
+	var toUnmount []string
+	for _, m := range mounts {
+		if strings.HasPrefix(m.Mountpoint, tempMountLocation) {
+			toUnmount = append(toUnmount, m.Mountpoint)
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(toUnmount)))
+	for _, path := range toUnmount {
+		if err := UnmountAll(path, flags); err != nil {
+			return err
+		}
+		if err := os.Remove(path); err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/github.com/containerd/containerd/mount/temp_unsupported.go

@@ -0,0 +1,13 @@
+// +build windows
+
+package mount
+
+// SetTempMountLocation sets the temporary mount location
+func SetTempMountLocation(root string) error {
+	return nil
+}
+
+// CleanupTempMounts all temp mounts and remove the directories
+func CleanupTempMounts(flags int) error {
+	return nil
+}

vendor/github.com/containerd/containerd/task.go

@@ -283,7 +283,6 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat
 	if t.io != nil {
 		t.io.Cancel()
 		t.io.Wait()
-		t.io.Close()
 	}
 	r, err := t.client.TaskService().Delete(ctx, &tasks.DeleteTaskRequest{
 		ContainerID: t.id,
@@ -291,6 +290,10 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat
 	if err != nil {
 		return nil, errdefs.FromGRPC(err)
 	}
+	// Only cleanup the IO after a successful Delete
+	if t.io != nil {
+		t.io.Close()
+	}
 	return &ExitStatus{code: r.ExitStatus, exitedAt: r.ExitedAt}, nil
 }
 

vendor/github.com/containerd/containerd/vendor.conf

@@ -42,3 +42,35 @@ github.com/stevvooe/ttrpc d4528379866b0ce7e9d71f3eb96f0582fc374577
 github.com/syndtr/gocapability db04d3cc01c8b54962a58ec7e491717d06cfcc16
 github.com/gotestyourself/gotestyourself 44dbf532bbf5767611f6f2a61bded572e337010a
 github.com/google/go-cmp v0.1.0
+# cri dependencies
+github.com/containerd/cri-containerd c9081b2ec0eefc799f0f1caabbea29d516c72c44
+github.com/blang/semver v3.1.0
+github.com/containernetworking/cni v0.6.0
+github.com/containernetworking/plugins v0.6.0
+github.com/cri-o/ocicni 9b451e26eb7c694d564991fbf44f77d0afb9b03c
+github.com/davecgh/go-spew v1.1.0
+github.com/docker/distribution b38e5838b7b2f2ad48e06ec4b500011976080621
+github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00
+github.com/docker/spdystream 449fdfce4d962303d702fec724ef0ad181c92528
+github.com/emicklei/go-restful ff4f55a206334ef123e4f79bbf348980da81ca46
+github.com/fsnotify/fsnotify 7d7316ed6e1ed2de075aab8dfc76de5d158d66e1
+github.com/ghodss/yaml 73d445a93680fa1a78ae23a5839bad48f32ba1ee
+github.com/golang/glog 44145f04b68cf362d9c4df2182967c2275eaefed
+github.com/google/gofuzz 44d81051d367757e1c7c6a5a86423ece9afcf63c
+github.com/hashicorp/errwrap 7554cd9344cec97297fa6649b055a8c98c2a1e55
+github.com/hashicorp/go-multierror ed905158d87462226a13fe39ddf685ea65f1c11f
+github.com/json-iterator/go 1.0.4
+github.com/opencontainers/runtime-tools 6073aff4ac61897f75895123f7e24135204a404d
+github.com/opencontainers/selinux 4a2974bf1ee960774ffd517717f1f45325af0206
+github.com/seccomp/libseccomp-golang 32f571b70023028bd57d9288c20efbcb237f3ce0
+github.com/spf13/pflag v1.0.0
+github.com/tchap/go-patricia 5ad6cdb7538b0097d5598c7e57f0a24072adf7dc
+golang.org/x/time f51c12702a4d776e4c1fa9b0fabab841babae631
+gopkg.in/inf.v0 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4
+gopkg.in/yaml.v2 53feefa2559fb8dfa8d81baad31be332c97d6c77
+k8s.io/api a1d6dce6736a6c75929bb75111e89077e35a5856
+k8s.io/apimachinery 8259d997cf059cd83dc47e5f8074b7a7d7967c09
+k8s.io/apiserver 8e45eac9dff86447a5c2effe6a3d2cba70121ebf
+k8s.io/client-go 33bd23f75b6de861994706a322b0afab824b2171
+k8s.io/kubernetes 05944b1d2ca7f60b09762a330425108f48f6b603
+k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e