Merge pull request #697 from Random-Liu/fs-layout-change

adds volatile state directory to the fs plan for cntrs/pods/fifo

cri.go

@@ -64,12 +64,10 @@ func initCRIService(ic *plugin.InitContext) (interface{}, error) {
 	pluginConfig := ic.Config.(*criconfig.PluginConfig)
 	c := criconfig.Config{
 		PluginConfig:       *pluginConfig,
-		// This is a hack. We assume that containerd root directory
-		// is one level above plugin directory.
-		// TODO(random-liu): Expose containerd config to plugin.
 		ContainerdRootDir:  filepath.Dir(ic.Root),
 		ContainerdEndpoint: ic.Address,
 		RootDir:            ic.Root,
+		StateDir:           ic.State,
 	}
 	log.G(ctx).Infof("Start cri plugin with config %+v", c)
 

docs/crictl.md

@@ -189,7 +189,8 @@ $ crictl info
     "statsCollectPeriod": 10,
     "containerdRootDir": "/var/lib/containerd",
     "containerdEndpoint": "/run/containerd/containerd.sock",
-    "rootDir": "/var/lib/containerd/io.containerd.grpc.v1.cri"
+    "rootDir": "/var/lib/containerd/io.containerd.grpc.v1.cri",
+    "stateDir": "/run/containerd/io.containerd.grpc.v1.cri",
   },
   "golang": "go1.10"
 }

pkg/config/config.go

@@ -96,6 +96,8 @@ type Config struct {
 	// RootDir is the root directory path for managing cri plugin files
 	// (metadata checkpoint etc.)
 	RootDir string `json:"rootDir"`
+	// StateDir is the root directory path for managing volatile pod/container data
+	StateDir string `json:"stateDir"`
 }
 
 // DefaultConfig returns default configurations of cri plugin.

pkg/os/os.go

@@ -25,6 +25,7 @@ import (
 	containerdmount "github.com/containerd/containerd/mount"
 	"github.com/containerd/fifo"
 	"github.com/docker/docker/pkg/mount"
+	"github.com/docker/docker/pkg/symlink"
 	"golang.org/x/net/context"
 	"golang.org/x/sys/unix"
 )
@@ -37,6 +38,7 @@ type OS interface {
 	OpenFifo(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error)
 	Stat(name string) (os.FileInfo, error)
 	ResolveSymbolicLink(name string) (string, error)
+	FollowSymlinkInScope(path, scope string) (string, error)
 	CopyFile(src, dest string, perm os.FileMode) error
 	WriteFile(filename string, data []byte, perm os.FileMode) error
 	Mount(source string, target string, fstype string, flags uintptr, data string) error
@@ -79,6 +81,11 @@ func (RealOS) ResolveSymbolicLink(path string) (string, error) {
 	return filepath.EvalSymlinks(path)
 }
 
+// FollowSymlinkInScope will call symlink.FollowSymlinkInScope.
+func (RealOS) FollowSymlinkInScope(path, scope string) (string, error) {
+	return symlink.FollowSymlinkInScope(path, scope)
+}
+
 // CopyFile will copy src file to dest file
 func (RealOS) CopyFile(src, dest string, perm os.FileMode) error {
 	in, err := os.Open(src)
@@ -107,17 +114,21 @@ func (RealOS) Mount(source string, target string, fstype string, flags uintptr,
 	return unix.Mount(source, target, fstype, flags, data)
 }
 
-// Unmount will call unix.Unmount to unmount the file. The function doesn't
+// Unmount will call Unmount to unmount the file.
-// return error if target is not mounted.
 func (RealOS) Unmount(target string, flags int) error {
-	// TODO(random-liu): Follow symlink to make sure the result is correct.
+	return Unmount(target, flags)
-	if mounted, err := mount.Mounted(target); err != nil || !mounted {
-		return err
-	}
-	return unix.Unmount(target, flags)
 }
 
 // LookupMount gets mount info of a given path.
 func (RealOS) LookupMount(path string) (containerdmount.Info, error) {
 	return containerdmount.Lookup(path)
 }
+
+// Unmount will call unix.Unmount to unmount the file. The function doesn't
+// return error if target is not mounted.
+func Unmount(target string, flags int) error {
+	if mounted, err := mount.Mounted(target); err != nil || !mounted {
+		return err
+	}
+	return unix.Unmount(target, flags)
+}

pkg/os/testing/fake_os.go

@@ -45,6 +45,7 @@ type FakeOS struct {
 	OpenFifoFn             func(context.Context, string, int, os.FileMode) (io.ReadWriteCloser, error)
 	StatFn                 func(string) (os.FileInfo, error)
 	ResolveSymbolicLinkFn  func(string) (string, error)
+	FollowSymlinkInScopeFn func(string, string) (string, error)
 	CopyFileFn             func(string, string, os.FileMode) error
 	WriteFileFn            func(string, []byte, os.FileMode) error
 	MountFn                func(source string, target string, fstype string, flags uintptr, data string) error
@@ -176,6 +177,19 @@ func (f *FakeOS) ResolveSymbolicLink(path string) (string, error) {
 	return path, nil
 }
 
+// FollowSymlinkInScope is a fake call that invokes FollowSymlinkInScope or returns its input
+func (f *FakeOS) FollowSymlinkInScope(path, scope string) (string, error) {
+	f.appendCalls("FollowSymlinkInScope", path, scope)
+	if err := f.getError("FollowSymlinkInScope"); err != nil {
+		return "", err
+	}
+
+	if f.FollowSymlinkInScopeFn != nil {
+		return f.FollowSymlinkInScopeFn(path, scope)
+	}
+	return path, nil
+}
+
 // CopyFile is a fake call that invokes CopyFileFn or just return nil.
 func (f *FakeOS) CopyFile(src, dest string, perm os.FileMode) error {
 	f.appendCalls("CopyFile", src, dest, perm)

pkg/server/container_create.go

@@ -134,7 +134,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	logrus.Debugf("Use OCI %+v for container %q", ociRuntime, id)
 
 	// Create container root directory.
-	containerRootDir := getContainerRootDir(c.config.RootDir, id)
+	containerRootDir := c.getContainerRootDir(id)
 	if err = c.os.MkdirAll(containerRootDir, 0755); err != nil {
 		return nil, errors.Wrapf(err, "failed to create container root directory %q",
 			containerRootDir)
@@ -148,12 +148,26 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 			}
 		}
 	}()
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err = c.os.MkdirAll(volatileContainerRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create volatile container root directory %q",
+			volatileContainerRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile container root directory.
+			if err = c.os.RemoveAll(volatileContainerRootDir); err != nil {
+				logrus.WithError(err).Errorf("Failed to remove volatile container root directory %q",
+					volatileContainerRootDir)
+			}
+		}
+	}()
 
 	// Create container volumes mounts.
 	volumeMounts := c.generateVolumeMounts(containerRootDir, config.GetMounts(), &image.ImageSpec.Config)
 
 	// Generate container runtime spec.
-	mounts := c.generateContainerMounts(getSandboxRootDir(c.config.RootDir, sandboxID), config)
+	mounts := c.generateContainerMounts(sandboxID, config)
 
 	spec, err := c.generateContainerSpec(id, sandboxID, sandboxPid, config, sandboxConfig, &image.ImageSpec.Config, append(mounts, volumeMounts...))
 	if err != nil {
@@ -188,7 +202,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
 	}
 
 	containerIO, err := cio.NewContainerIO(id,
-		cio.WithNewFIFOs(containerRootDir, config.GetTty(), config.GetStdin()))
+		cio.WithNewFIFOs(volatileContainerRootDir, config.GetTty(), config.GetStdin()))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create container io")
 	}
@@ -416,13 +430,13 @@ func (c *criService) generateVolumeMounts(containerRootDir string, criMounts []*
 
 // generateContainerMounts sets up necessary container mounts including /dev/shm, /etc/hosts
 // and /etc/resolv.conf.
-func (c *criService) generateContainerMounts(sandboxRootDir string, config *runtime.ContainerConfig) []*runtime.Mount {
+func (c *criService) generateContainerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
 	var mounts []*runtime.Mount
 	securityContext := config.GetLinux().GetSecurityContext()
 	if !isInCRIMounts(etcHosts, config.GetMounts()) {
 		mounts = append(mounts, &runtime.Mount{
 			ContainerPath: etcHosts,
-			HostPath:      getSandboxHosts(sandboxRootDir),
+			HostPath:      c.getSandboxHosts(sandboxID),
 			Readonly:      securityContext.GetReadonlyRootfs(),
 		})
 	}
@@ -432,13 +446,13 @@ func (c *criService) generateContainerMounts(sandboxRootDir string, config *runt
 	if !isInCRIMounts(resolvConfPath, config.GetMounts()) {
 		mounts = append(mounts, &runtime.Mount{
 			ContainerPath: resolvConfPath,
-			HostPath:      getResolvPath(sandboxRootDir),
+			HostPath:      c.getResolvPath(sandboxID),
 			Readonly:      securityContext.GetReadonlyRootfs(),
 		})
 	}
 
 	if !isInCRIMounts(devShm, config.GetMounts()) {
-		sandboxDevShm := getSandboxDevShm(sandboxRootDir)
+		sandboxDevShm := c.getSandboxDevShm(sandboxID)
 		if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
 			sandboxDevShm = devShm
 		}

pkg/server/container_create_test.go

@@ -497,7 +497,7 @@ func TestGenerateVolumeMounts(t *testing.T) {
 }
 
 func TestGenerateContainerMounts(t *testing.T) {
-	testSandboxRootDir := "test-sandbox-root"
+	const testSandboxID = "test-id"
 	for desc, test := range map[string]struct {
 		criMounts       []*runtime.Mount
 		securityContext *runtime.LinuxContainerSecurityContext
@@ -510,17 +510,17 @@ func TestGenerateContainerMounts(t *testing.T) {
 			expectedMounts: []*runtime.Mount{
 				{
 					ContainerPath: "/etc/hosts",
-					HostPath:      testSandboxRootDir + "/hosts",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
 					Readonly:      true,
 				},
 				{
 					ContainerPath: resolvConfPath,
-					HostPath:      testSandboxRootDir + "/resolv.conf",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
 					Readonly:      true,
 				},
 				{
 					ContainerPath: "/dev/shm",
-					HostPath:      testSandboxRootDir + "/shm",
+					HostPath:      filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
 					Readonly:      false,
 				},
 			},
@@ -530,17 +530,17 @@ func TestGenerateContainerMounts(t *testing.T) {
 			expectedMounts: []*runtime.Mount{
 				{
 					ContainerPath: "/etc/hosts",
-					HostPath:      testSandboxRootDir + "/hosts",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
 					Readonly:      false,
 				},
 				{
 					ContainerPath: resolvConfPath,
-					HostPath:      testSandboxRootDir + "/resolv.conf",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
 					Readonly:      false,
 				},
 				{
 					ContainerPath: "/dev/shm",
-					HostPath:      testSandboxRootDir + "/shm",
+					HostPath:      filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
 					Readonly:      false,
 				},
 			},
@@ -552,12 +552,12 @@ func TestGenerateContainerMounts(t *testing.T) {
 			expectedMounts: []*runtime.Mount{
 				{
 					ContainerPath: "/etc/hosts",
-					HostPath:      testSandboxRootDir + "/hosts",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
 					Readonly:      false,
 				},
 				{
 					ContainerPath: resolvConfPath,
-					HostPath:      testSandboxRootDir + "/resolv.conf",
+					HostPath:      filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
 					Readonly:      false,
 				},
 				{
@@ -597,7 +597,7 @@ func TestGenerateContainerMounts(t *testing.T) {
 			},
 		}
 		c := newTestCRIService()
-		mounts := c.generateContainerMounts(testSandboxRootDir, config)
+		mounts := c.generateContainerMounts(testSandboxID, config)
 		assert.Equal(t, test.expectedMounts, mounts, desc)
 	}
 }

pkg/server/container_execsync.go

@@ -116,12 +116,12 @@ func (c *criService) execInContainer(ctx context.Context, id string, opts execOp
 	}
 	execID := util.GenerateID()
 	logrus.Debugf("Generated exec id %q for container %q", execID, id)
-	rootDir := getContainerRootDir(c.config.RootDir, id)
+	volatileRootDir := c.getVolatileContainerRootDir(id)
 	var execIO *cio.ExecIO
 	process, err := task.Exec(ctx, execID, pspec,
 		func(id string) (containerdio.IO, error) {
 			var err error
-			execIO, err = cio.NewExecIO(id, rootDir, opts.tty, opts.stdin != nil)
+			execIO, err = cio.NewExecIO(id, volatileRootDir, opts.tty, opts.stdin != nil)
 			return execIO, err
 		},
 	)

pkg/server/container_remove.go

@@ -76,11 +76,16 @@ func (c *criService) RemoveContainer(ctx context.Context, r *runtime.RemoveConta
 		return nil, errors.Wrapf(err, "failed to delete container checkpoint for %q", id)
 	}
 
-	containerRootDir := getContainerRootDir(c.config.RootDir, id)
+	containerRootDir := c.getContainerRootDir(id)
 	if err := system.EnsureRemoveAll(containerRootDir); err != nil {
 		return nil, errors.Wrapf(err, "failed to remove container root directory %q",
 			containerRootDir)
 	}
+	volatileContainerRootDir := c.getVolatileContainerRootDir(id)
+	if err := system.EnsureRemoveAll(volatileContainerRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove volatile container root directory %q",
+			volatileContainerRootDir)
+	}
 
 	c.containerStore.Delete(id)
 

pkg/server/helpers.go

@@ -156,29 +156,42 @@ func getCgroupsPath(cgroupsParent, id string, systemdCgroup bool) string {
 }
 
 // getSandboxRootDir returns the root directory for managing sandbox files,
-// e.g. named pipes.
+// e.g. hosts files.
-func getSandboxRootDir(rootDir, id string) string {
+func (c *criService) getSandboxRootDir(id string) string {
-	return filepath.Join(rootDir, sandboxesDir, id)
+	return filepath.Join(c.config.RootDir, sandboxesDir, id)
 }
 
-// getContainerRootDir returns the root directory for managing container files.
+// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
-func getContainerRootDir(rootDir, id string) string {
+// e.g. named pipes.
-	return filepath.Join(rootDir, containersDir, id)
+func (c *criService) getVolatileSandboxRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, sandboxesDir, id)
+}
+
+// getContainerRootDir returns the root directory for managing container files,
+// e.g. state checkpoint.
+func (c *criService) getContainerRootDir(id string) string {
+	return filepath.Join(c.config.RootDir, containersDir, id)
+}
+
+// getVolatileContainerRootDir returns the root directory for managing volatile container files,
+// e.g. named pipes.
+func (c *criService) getVolatileContainerRootDir(id string) string {
+	return filepath.Join(c.config.StateDir, containersDir, id)
 }
 
 // getSandboxHosts returns the hosts file path inside the sandbox root directory.
-func getSandboxHosts(sandboxRootDir string) string {
+func (c *criService) getSandboxHosts(id string) string {
-	return filepath.Join(sandboxRootDir, "hosts")
+	return filepath.Join(c.getSandboxRootDir(id), "hosts")
 }
 
 // getResolvPath returns resolv.conf filepath for specified sandbox.
-func getResolvPath(sandboxRoot string) string {
+func (c *criService) getResolvPath(id string) string {
-	return filepath.Join(sandboxRoot, "resolv.conf")
+	return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
 }
 
 // getSandboxDevShm returns the shm file path inside the sandbox root directory.
-func getSandboxDevShm(sandboxRootDir string) string {
+func (c *criService) getSandboxDevShm(id string) string {
-	return filepath.Join(sandboxRootDir, "shm")
+	return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
 }
 
 // getNetworkNamespace returns the network namespace of a process.

pkg/server/restart.go

@@ -78,8 +78,9 @@ func (c *criService) recover(ctx context.Context) error {
 		return errors.Wrap(err, "failed to list containers")
 	}
 	for _, container := range containers {
-		containerDir := getContainerRootDir(c.config.RootDir, container.ID())
+		containerDir := c.getContainerRootDir(container.ID())
-		cntr, err := loadContainer(ctx, container, containerDir)
+		volatileContainerDir := c.getVolatileContainerRootDir(container.ID())
+		cntr, err := loadContainer(ctx, container, containerDir, volatileContainerDir)
 		if err != nil {
 			logrus.WithError(err).Errorf("Failed to load container %q", container.ID())
 			continue
@@ -113,21 +114,42 @@ func (c *criService) recover(ctx context.Context) error {
 	// we can't even get metadata, we should cleanup orphaned sandbox/container directories
 	// with best effort.
 
-	// Cleanup orphaned sandbox directories without corresponding containerd container.
+	// Cleanup orphaned sandbox and container directories without corresponding containerd container.
-	if err := cleanupOrphanedSandboxDirs(sandboxes, filepath.Join(c.config.RootDir, "sandboxes")); err != nil {
+	for _, cleanup := range []struct {
-		return errors.Wrap(err, "failed to cleanup orphaned sandbox directories")
+		cntrs  []containerd.Container
+		base   string
+		errMsg string
+	}{
+		{
+			cntrs:  sandboxes,
+			base:   filepath.Join(c.config.RootDir, sandboxesDir),
+			errMsg: "failed to cleanup orphaned sandbox directories",
+		},
+		{
+			cntrs:  sandboxes,
+			base:   filepath.Join(c.config.StateDir, sandboxesDir),
+			errMsg: "failed to cleanup orphaned volatile sandbox directories",
+		},
+		{
+			cntrs:  containers,
+			base:   filepath.Join(c.config.RootDir, containersDir),
+			errMsg: "failed to cleanup orphaned container directories",
+		},
+		{
+			cntrs:  containers,
+			base:   filepath.Join(c.config.StateDir, containersDir),
+			errMsg: "failed to cleanup orphaned volatile container directories",
+		},
+	} {
+		if err := cleanupOrphanedIDDirs(cleanup.cntrs, cleanup.base); err != nil {
+			return errors.Wrap(err, cleanup.errMsg)
 		}
-
-	// Cleanup orphaned container directories without corresponding containerd container.
-	if err := cleanupOrphanedContainerDirs(containers, filepath.Join(c.config.RootDir, "containers")); err != nil {
-		return errors.Wrap(err, "failed to cleanup orphaned container directories")
 	}
-
 	return nil
 }
 
 // loadContainer loads container from containerd and status checkpoint.
-func loadContainer(ctx context.Context, cntr containerd.Container, containerDir string) (containerstore.Container, error) {
+func loadContainer(ctx context.Context, cntr containerd.Container, containerDir, volatileContainerDir string) (containerstore.Container, error) {
 	id := cntr.ID()
 	var container containerstore.Container
 	// Load container metadata.
@@ -197,7 +219,7 @@ func loadContainer(ctx context.Context, cntr containerd.Container, containerDir
 			// containerd got restarted during that. In that case, we still
 			// treat the container as `CREATED`.
 			containerIO, err = cio.NewContainerIO(id,
-				cio.WithNewFIFOs(containerDir, meta.Config.GetTty(), meta.Config.GetStdin()),
+				cio.WithNewFIFOs(volatileContainerDir, meta.Config.GetTty(), meta.Config.GetStdin()),
 			)
 			if err != nil {
 				return container, errors.Wrap(err, "failed to create container io")
@@ -448,59 +470,30 @@ func loadImages(ctx context.Context, cImages []containerd.Image,
 	return images, nil
 }
 
-func cleanupOrphanedSandboxDirs(cntrs []containerd.Container, sandboxesRoot string) error {
+func cleanupOrphanedIDDirs(cntrs []containerd.Container, base string) error {
-	// Cleanup orphaned sandbox directories.
+	// Cleanup orphaned id directories.
-	dirs, err := ioutil.ReadDir(sandboxesRoot)
+	dirs, err := ioutil.ReadDir(base)
 	if err != nil && !os.IsNotExist(err) {
-		return errors.Wrapf(err, "failed to read pod sandboxes directory %q", sandboxesRoot)
+		return errors.Wrap(err, "failed to read base directory")
 	}
-	cntrsMap := make(map[string]containerd.Container)
+	idsMap := make(map[string]containerd.Container)
 	for _, cntr := range cntrs {
-		cntrsMap[cntr.ID()] = cntr
+		idsMap[cntr.ID()] = cntr
 	}
 	for _, d := range dirs {
 		if !d.IsDir() {
-			logrus.Warnf("Invalid file %q found in pod sandboxes directory", d.Name())
+			logrus.Warnf("Invalid file %q found in base directory %q", d.Name(), base)
 			continue
 		}
-		if _, ok := cntrsMap[d.Name()]; ok {
+		if _, ok := idsMap[d.Name()]; ok {
-			// Do not remove sandbox directory if corresponding container is found.
+			// Do not remove id directory if corresponding container is found.
 			continue
 		}
-		sandboxDir := filepath.Join(sandboxesRoot, d.Name())
+		dir := filepath.Join(base, d.Name())
-		if err := system.EnsureRemoveAll(sandboxDir); err != nil {
+		if err := system.EnsureRemoveAll(dir); err != nil {
-			logrus.WithError(err).Warnf("Failed to remove pod sandbox directory %q", sandboxDir)
+			logrus.WithError(err).Warnf("Failed to remove id directory %q", dir)
 		} else {
-			logrus.Debugf("Cleanup orphaned pod sandbox directory %q", sandboxDir)
+			logrus.Debugf("Cleanup orphaned id directory %q", dir)
-		}
-	}
-	return nil
-}
-
-func cleanupOrphanedContainerDirs(cntrs []containerd.Container, containersRoot string) error {
-	// Cleanup orphaned container directories.
-	dirs, err := ioutil.ReadDir(containersRoot)
-	if err != nil && !os.IsNotExist(err) {
-		return errors.Wrapf(err, "failed to read containers directory %q", containersRoot)
-	}
-	cntrsMap := make(map[string]containerd.Container)
-	for _, cntr := range cntrs {
-		cntrsMap[cntr.ID()] = cntr
-	}
-	for _, d := range dirs {
-		if !d.IsDir() {
-			logrus.Warnf("Invalid file %q found in containers directory", d.Name())
-			continue
-		}
-		if _, ok := cntrsMap[d.Name()]; ok {
-			// Do not remove container directory if corresponding container is found.
-			continue
-		}
-		containerDir := filepath.Join(containersRoot, d.Name())
-		if err := system.EnsureRemoveAll(containerDir); err != nil {
-			logrus.WithError(err).Warnf("Failed to remove container directory %q", containerDir)
-		} else {
-			logrus.Debugf("Cleanup orphaned container directory %q", containerDir)
 		}
 	}
 	return nil

pkg/server/sandbox_remove.go

@@ -72,12 +72,17 @@ func (c *criService) RemovePodSandbox(ctx context.Context, r *runtime.RemovePodS
 		}
 	}
 
-	// Cleanup the sandbox root directory.
+	// Cleanup the sandbox root directories.
-	sandboxRootDir := getSandboxRootDir(c.config.RootDir, id)
+	sandboxRootDir := c.getSandboxRootDir(id)
 	if err := system.EnsureRemoveAll(sandboxRootDir); err != nil {
 		return nil, errors.Wrapf(err, "failed to remove sandbox root directory %q",
 			sandboxRootDir)
 	}
+	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
+	if err := system.EnsureRemoveAll(volatileSandboxRootDir); err != nil {
+		return nil, errors.Wrapf(err, "failed to remove volatile sandbox root directory %q",
+			volatileSandboxRootDir)
+	}
 
 	// Delete sandbox container.
 	if err := sandbox.Container.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {

pkg/server/sandbox_run.go

@@ -189,8 +189,8 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 		}
 	}()
 
-	// Create sandbox container root directory.
+	// Create sandbox container root directories.
-	sandboxRootDir := getSandboxRootDir(c.config.RootDir, id)
+	sandboxRootDir := c.getSandboxRootDir(id)
 	if err := c.os.MkdirAll(sandboxRootDir, 0755); err != nil {
 		return nil, errors.Wrapf(err, "failed to create sandbox root directory %q",
 			sandboxRootDir)
@@ -204,14 +204,28 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
 			}
 		}
 	}()
+	volatileSandboxRootDir := c.getVolatileSandboxRootDir(id)
+	if err := c.os.MkdirAll(volatileSandboxRootDir, 0755); err != nil {
+		return nil, errors.Wrapf(err, "failed to create volatile sandbox root directory %q",
+			volatileSandboxRootDir)
+	}
+	defer func() {
+		if retErr != nil {
+			// Cleanup the volatile sandbox root directory.
+			if err := c.os.RemoveAll(volatileSandboxRootDir); err != nil {
+				logrus.WithError(err).Errorf("Failed to remove volatile sandbox root directory %q",
+					volatileSandboxRootDir)
+			}
+		}
+	}()
 
 	// Setup sandbox /dev/shm, /etc/hosts and /etc/resolv.conf.
-	if err = c.setupSandboxFiles(sandboxRootDir, config); err != nil {
+	if err = c.setupSandboxFiles(id, config); err != nil {
 		return nil, errors.Wrapf(err, "failed to setup sandbox files")
 	}
 	defer func() {
 		if retErr != nil {
-			if err = c.unmountSandboxFiles(sandboxRootDir, config); err != nil {
+			if err = c.unmountSandboxFiles(id, config); err != nil {
 				logrus.WithError(err).Errorf("Failed to unmount sandbox files in %q",
 					sandboxRootDir)
 			}
@@ -398,9 +412,9 @@ func (c *criService) generateSandboxContainerSpec(id string, config *runtime.Pod
 
 // setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts
 // and /etc/resolv.conf.
-func (c *criService) setupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) error {
+func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
 	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
-	sandboxEtcHosts := getSandboxHosts(rootDir)
+	sandboxEtcHosts := c.getSandboxHosts(id)
 	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
 		return errors.Wrapf(err, "failed to generate sandbox hosts file %q", sandboxEtcHosts)
 	}
@@ -414,7 +428,7 @@ func (c *criService) setupSandboxFiles(rootDir string, config *runtime.PodSandbo
 			return errors.Wrapf(err, "failed to parse sandbox DNSConfig %+v", dnsConfig)
 		}
 	}
-	resolvPath := getResolvPath(rootDir)
+	resolvPath := c.getResolvPath(id)
 	if resolvContent == "" {
 		// copy host's resolv.conf to resolvPath
 		err = c.os.CopyFile(resolvConfPath, resolvPath, 0644)
@@ -434,7 +448,7 @@ func (c *criService) setupSandboxFiles(rootDir string, config *runtime.PodSandbo
 			return errors.Wrapf(err, "host %q is not available for host ipc", devShm)
 		}
 	} else {
-		sandboxDevShm := getSandboxDevShm(rootDir)
+		sandboxDevShm := c.getSandboxDevShm(id)
 		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
 			return errors.Wrap(err, "failed to create sandbox shm")
 		}
@@ -475,10 +489,14 @@ func parseDNSOptions(servers, searches, options []string) (string, error) {
 // remove these files. Unmount should *NOT* return error when:
 //  1) The mount point is already unmounted.
 //  2) The mount point doesn't exist.
-func (c *criService) unmountSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) error {
+func (c *criService) unmountSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
 	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() != runtime.NamespaceMode_NODE {
-		if err := c.os.Unmount(getSandboxDevShm(rootDir), unix.MNT_DETACH); err != nil && !os.IsNotExist(err) {
+		path, err := c.os.FollowSymlinkInScope(c.getSandboxDevShm(id), "/")
-			return err
+		if err != nil {
+			return errors.Wrap(err, "failed to follow symlink")
+		}
+		if err := c.os.Unmount(path, unix.MNT_DETACH); err != nil && !os.IsNotExist(err) {
+			return errors.Wrapf(err, "failed to unmount %q", path)
 		}
 	}
 	return nil

pkg/server/sandbox_run_test.go

@@ -18,6 +18,7 @@ package server
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	cni "github.com/containerd/go-cni"
@@ -177,7 +178,7 @@ func TestGenerateSandboxContainerSpec(t *testing.T) {
 }
 
 func TestSetupSandboxFiles(t *testing.T) {
-	testRootDir := "test-sandbox-root"
+	const testID = "test-id"
 	for desc, test := range map[string]struct {
 		dnsConfig     *runtime.DNSConfig
 		ipcMode       runtime.NamespaceMode
@@ -189,13 +190,17 @@ func TestSetupSandboxFiles(t *testing.T) {
 				{
 					Name: "CopyFile",
 					Arguments: []interface{}{
-						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
 					},
 				},
 				{
 					Name: "CopyFile",
 					Arguments: []interface{}{
-						"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
+						"/etc/resolv.conf",
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						os.FileMode(0644),
 					},
 				},
 				{
@@ -215,13 +220,16 @@ func TestSetupSandboxFiles(t *testing.T) {
 				{
 					Name: "CopyFile",
 					Arguments: []interface{}{
-						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
 					},
 				},
 				{
 					Name: "WriteFile",
 					Arguments: []interface{}{
-						testRootDir + "/resolv.conf", []byte(`search 114.114.114.114
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						[]byte(`search 114.114.114.114
 nameserver 8.8.8.8
 options timeout:1
 `), os.FileMode(0644),
@@ -239,19 +247,24 @@ options timeout:1
 				{
 					Name: "CopyFile",
 					Arguments: []interface{}{
-						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+						"/etc/hosts",
+						filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
+						os.FileMode(0644),
 					},
 				},
 				{
 					Name: "CopyFile",
 					Arguments: []interface{}{
-						"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
+						"/etc/resolv.conf",
+						filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
+						os.FileMode(0644),
 					},
 				},
 				{
 					Name: "MkdirAll",
 					Arguments: []interface{}{
-						testRootDir + "/shm", os.FileMode(0700),
+						filepath.Join(testStateDir, sandboxesDir, testID, "shm"),
+						os.FileMode(0700),
 					},
 				},
 				{
@@ -273,7 +286,7 @@ options timeout:1
 				},
 			},
 		}
-		c.setupSandboxFiles(testRootDir, cfg)
+		c.setupSandboxFiles(testID, cfg)
 		calls := c.os.(*ostesting.FakeOS).GetCalls()
 		assert.Len(t, calls, len(test.expectedCalls))
 		for i, expected := range test.expectedCalls {

pkg/server/sandbox_stop.go

@@ -81,9 +81,8 @@ func (c *criService) StopPodSandbox(ctx context.Context, r *runtime.StopPodSandb
 
 	logrus.Infof("TearDown network for sandbox %q successfully", id)
 
-	sandboxRoot := getSandboxRootDir(c.config.RootDir, id)
+	if err := c.unmountSandboxFiles(id, sandbox.Config); err != nil {
-	if err := c.unmountSandboxFiles(sandboxRoot, sandbox.Config); err != nil {
+		return nil, errors.Wrap(err, "failed to unmount sandbox files")
-		return nil, errors.Wrapf(err, "failed to unmount sandbox files in %q", sandboxRoot)
 	}
 
 	// Only stop sandbox container when it's running.

pkg/server/service_test.go

@@ -28,7 +28,8 @@ import (
 )
 
 const (
-	testRootDir = "/test/rootfs"
+	testRootDir  = "/test/root"
+	testStateDir = "/test/state"
 	// Use an image id as test sandbox image to avoid image name resolve.
 	// TODO(random-liu): Change this to image name after we have complete image
 	// management unit test framework.
@@ -41,6 +42,7 @@ func newTestCRIService() *criService {
 	return &criService{
 		config: criconfig.Config{
 			RootDir:  testRootDir,
+			StateDir: testStateDir,
 			PluginConfig: criconfig.PluginConfig{
 				SandboxImage: testSandboxImage,
 			},

pkg/store/sandbox/netns.go

@@ -21,10 +21,11 @@ import (
 	"sync"
 
 	cnins "github.com/containernetworking/plugins/pkg/ns"
-	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/symlink"
 	"github.com/pkg/errors"
 	"golang.org/x/sys/unix"
+
+	osinterface "github.com/containerd/cri/pkg/os"
 )
 
 // ErrClosedNetNS is the error returned when network namespace is closed.
@@ -81,7 +82,6 @@ func (n *NetNS) Remove() error {
 	}
 	if n.restored {
 		path := n.ns.Path()
-		// TODO(random-liu): Add util function for unmount.
 		// Check netns existence.
 		if _, err := os.Stat(path); err != nil {
 			if os.IsNotExist(err) {
@@ -93,16 +93,9 @@ func (n *NetNS) Remove() error {
 		if err != nil {
 			return errors.Wrap(err, "failed to follow symlink")
 		}
-		mounted, err := mount.Mounted(path)
+		if err := osinterface.Unmount(path, unix.MNT_DETACH); err != nil && !os.IsNotExist(err) {
-		if err != nil {
-			return errors.Wrap(err, "failed to check netns mounted")
-		}
-		if mounted {
-			err := unix.Unmount(path, unix.MNT_DETACH)
-			if err != nil && !os.IsNotExist(err) {
 			return errors.Wrap(err, "failed to umount netns")
 		}
-		}
 		if err := os.RemoveAll(path); err != nil {
 			return errors.Wrap(err, "failed to remove netns")
 		}