Add sandbox /dev/shm.

Signed-off-by: Lantao Liu <lantaol@google.com>

pkg/os/testing/fake_os.go

@@ -183,6 +183,7 @@ func (f *FakeOS) WriteFile(filename string, data []byte, perm os.FileMode) error
 	if f.WriteFileFn != nil {
 		return f.WriteFileFn(filename, data, perm)
 	}
+	return nil
 }
 
 // Mount is a fake call that invokes MountFn or just return nil.

pkg/server/container_start.go

@@ -315,7 +315,15 @@ func (c *criContainerdService) generateContainerMounts(sandboxRootDir string, co
 		Readonly:      securityContext.GetReadonlyRootfs(),
 	})
 
-	// TODO(random-liu): [P0] Mount sandbox /dev/shm.
+	sandboxDevShm := getSandboxDevShm(sandboxRootDir)
+	if securityContext.GetNamespaceOptions().GetHostIpc() {
+		sandboxDevShm = devShm
+	}
+	mounts = append(mounts, &runtime.Mount{
+		ContainerPath: devShm,
+		HostPath:      sandboxDevShm,
+		Readonly:      false,
+	})
 	return mounts
 }
 
@@ -416,6 +424,8 @@ func addOCIDevices(g *generate.Generator, devs []*runtime.Device, privileged boo
 }
 
 // addOCIBindMounts adds bind mounts.
+// TODO(random-liu): Figure out whether we need to change all CRI mounts to readonly when
+// rootfs is readonly. (https://github.com/moby/moby/blob/master/daemon/oci_linux.go)
 func addOCIBindMounts(g *generate.Generator, mounts []*runtime.Mount, privileged bool) {
 	for _, mount := range mounts {
 		dst := mount.GetContainerPath()

pkg/server/container_start_test.go

@@ -324,6 +324,11 @@ func TestGenerateContainerMounts(t *testing.T) {
 					HostPath:      testSandboxRootDir + "/resolv.conf",
 					Readonly:      true,
 				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      testSandboxRootDir + "/shm",
+					Readonly:      false,
+				},
 			},
 		},
 		"should setup rw mount when rootfs is read-write": {
@@ -336,7 +341,34 @@ func TestGenerateContainerMounts(t *testing.T) {
 				},
 				{
 					ContainerPath: resolvConfPath,
-					HostPath:      getResolvPath(testSandboxRootDir),
+					HostPath:      testSandboxRootDir + "/resolv.conf",
+					Readonly:      false,
+				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      testSandboxRootDir + "/shm",
+					Readonly:      false,
+				},
+			},
+		},
+		"should use host /dev/shm when host ipc is set": {
+			securityContext: &runtime.LinuxContainerSecurityContext{
+				NamespaceOptions: &runtime.NamespaceOption{HostIpc: true},
+			},
+			expectedMounts: []*runtime.Mount{
+				{
+					ContainerPath: "/etc/hosts",
+					HostPath:      testSandboxRootDir + "/hosts",
+					Readonly:      false,
+				},
+				{
+					ContainerPath: resolvConfPath,
+					HostPath:      testSandboxRootDir + "/resolv.conf",
+					Readonly:      false,
+				},
+				{
+					ContainerPath: "/dev/shm",
+					HostPath:      "/dev/shm",
 					Readonly:      false,
 				},
 			},

pkg/server/helpers.go

@@ -58,6 +58,8 @@ const (
 	// defaultSandboxImage is the image used by sandbox container.
 	// TODO(random-liu): [P1] Build schema 2 pause image and use it here.
 	defaultSandboxImage = "gcr.io/google.com/noogler-kubernetes/pause-amd64:3.0"
+	// defaultShmSize is the default size of the sandbox shm.
+	defaultShmSize = int64(1024 * 1024 * 64)
 	// relativeRootfsPath is the rootfs path relative to bundle path.
 	relativeRootfsPath = "rootfs"
 	// defaultRuntime is the runtime to use in containerd. We may support
@@ -88,6 +90,8 @@ const (
 	utsNSFormat = "/proc/%v/ns/uts"
 	// pidNSFormat is the format of pid namespace of a process.
 	pidNSFormat = "/proc/%v/ns/pid"
+	// devShm is the default path of /dev/shm.
+	devShm = "/dev/shm"
 	// etcHosts is the default path of /etc/hosts file.
 	etcHosts = "/etc/hosts"
 	// resolvConfPath is the abs path of resolv.conf on host or container.
@@ -159,6 +163,11 @@ func getResolvPath(sandboxRoot string) string {
 	return filepath.Join(sandboxRoot, "resolv.conf")
 }
 
+// getSandboxDevShm returns the shm file path inside the sandbox root directory.
+func getSandboxDevShm(sandboxRootDir string) string {
+	return filepath.Join(sandboxRootDir, "shm")
+}
+
 // prepareStreamingPipes prepares stream named pipe for container. returns nil
 // streaming handler if corresponding stream path is empty.
 func (c *criContainerdService) prepareStreamingPipes(ctx context.Context, stdin, stdout, stderr string) (

pkg/server/sandbox_run.go

@@ -19,6 +19,7 @@ package server
 import (
 	"encoding/json"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -31,6 +32,7 @@ import (
 	runtimespec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"golang.org/x/net/context"
+	"golang.org/x/sys/unix"
 	runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
 
 	"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@@ -137,8 +139,11 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
 	if err = c.setupSandboxFiles(sandboxRootDir, config); err != nil {
 		return nil, fmt.Errorf("failed to setup sandbox files: %v", err)
 	}
-	// No need to cleanup on error, because the whole sandbox root directory will be removed
+	defer func() {
-	// on error.
+		if retErr != nil {
+			c.cleanupSandboxFiles(sandboxRootDir, config)
+		}
+	}()
 
 	// Start sandbox container.
 	spec, err := c.generateSandboxContainerSpec(id, config, imageMeta.Config)
@@ -301,7 +306,7 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
 func (c *criContainerdService) setupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) error {
 	// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
 	sandboxEtcHosts := getSandboxHosts(rootDir)
-	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0666); err != nil {
+	if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
 		return fmt.Errorf("failed to generate sandbox hosts file %q: %v", sandboxEtcHosts, err)
 	}
 
@@ -328,8 +333,22 @@ func (c *criContainerdService) setupSandboxFiles(rootDir string, config *runtime
 		}
 	}
 
-	// TODO(random-liu): [P0] Deal with /dev/shm. Use host for HostIpc, and create and mount for
+	// Setup sandbox /dev/shm.
-	// non-HostIpc. What about mqueue?
+	if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostIpc() {
+		if _, err := c.os.Stat(devShm); err != nil {
+			return fmt.Errorf("host %q is not available for host ipc: %v", devShm, err)
+		}
+	} else {
+		sandboxDevShm := getSandboxDevShm(rootDir)
+		if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
+			return fmt.Errorf("failed to create sandbox shm: %v", err)
+		}
+		shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
+		if err := c.os.Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
+			return fmt.Errorf("failed to mount sandbox shm: %v", err)
+		}
+	}
+
 	return nil
 }
 
@@ -356,3 +375,13 @@ func parseDNSOptions(servers, searches, options []string) (string, error) {
 
 	return resolvContent, nil
 }
+
+// cleanupSandboxFiles only unmount files, we rely on the removal of sandbox root directory to remove files.
+// Each cleanup task should log error instead of returning, so as to keep on cleanup on error.
+func (c *criContainerdService) cleanupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) {
+	if !config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostIpc() {
+		if err := c.os.Unmount(getSandboxDevShm(rootDir), unix.MNT_DETACH); err != nil && os.IsNotExist(err) {
+			glog.Errorf("failed to unmount sandbox shm: %v", err)
+		}
+	}
+}

pkg/server/sandbox_run_test.go

@@ -159,18 +159,112 @@ func TestGenerateSandboxContainerSpec(t *testing.T) {
 
 func TestSetupSandboxFiles(t *testing.T) {
 	testRootDir := "test-sandbox-root"
-	expectedCopys := [][]interface{}{
+	for desc, test := range map[string]struct {
-		{"/etc/hosts", testRootDir + "/hosts", os.FileMode(0666)},
+		dnsConfig     *runtime.DNSConfig
-		{"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644)},
+		hostIpc       bool
-	}
+		expectedCalls []ostesting.CalledDetail
+	}{
+		"should check host /dev/shm existence when hostIpc is true": {
+			hostIpc: true,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+		"should create new /etc/resolv.conf if DNSOptions is set": {
+			dnsConfig: &runtime.DNSConfig{
+				Servers:  []string{"8.8.8.8"},
+				Searches: []string{"114.114.114.114"},
+				Options:  []string{"timeout:1"},
+			},
+			hostIpc: true,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+					},
+				},
+				{
+					Name: "WriteFile",
+					Arguments: []interface{}{
+						testRootDir + "/resolv.conf", []byte(`search 114.114.114.114
+nameserver 8.8.8.8
+options timeout:1
+`), os.FileMode(0644),
+					},
+				},
+				{
+					Name:      "Stat",
+					Arguments: []interface{}{"/dev/shm"},
+				},
+			},
+		},
+		"should create sandbox shm when hostIpc is false": {
+			hostIpc: false,
+			expectedCalls: []ostesting.CalledDetail{
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
+					},
+				},
+				{
+					Name: "CopyFile",
+					Arguments: []interface{}{
+						"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
+					},
+				},
+				{
+					Name: "MkdirAll",
+					Arguments: []interface{}{
+						testRootDir + "/shm", os.FileMode(0700),
+					},
+				},
+				{
+					Name: "Mount",
+					// Ignore arguments which are too complex to check.
+				},
+			},
+		},
+	} {
+		t.Logf("TestCase %q", desc)
 		c := newTestCRIContainerdService()
-	var copys [][]interface{}
+		cfg := &runtime.PodSandboxConfig{
-	c.os.(*ostesting.FakeOS).CopyFileFn = func(src string, dest string, perm os.FileMode) error {
+			DnsConfig: test.dnsConfig,
-		copys = append(copys, []interface{}{src, dest, perm})
+			Linux: &runtime.LinuxPodSandboxConfig{
-		return nil
+				SecurityContext: &runtime.LinuxSandboxSecurityContext{
+					NamespaceOptions: &runtime.NamespaceOption{
+						HostIpc: test.hostIpc,
+					},
+				},
+			},
+		}
+		c.setupSandboxFiles(testRootDir, cfg)
+		calls := c.os.(*ostesting.FakeOS).GetCalls()
+		assert.Len(t, calls, len(test.expectedCalls))
+		for i, expected := range test.expectedCalls {
+			if expected.Arguments == nil {
+				// Ignore arguments.
+				expected.Arguments = calls[i].Arguments
+			}
+			assert.Equal(t, expected, calls[i])
+		}
 	}
-	c.setupSandboxFiles(testRootDir, nil)
-	assert.Equal(t, expectedCopys, copys, "should copy expected files for sandbox")
 }
 
 func TestRunPodSandbox(t *testing.T) {
@@ -184,7 +278,6 @@ func TestRunPodSandbox(t *testing.T) {
 	var pipes []string
 	fakeOS.MkdirAllFn = func(path string, perm os.FileMode) error {
 		dirs = append(dirs, path)
-		assert.Equal(t, os.FileMode(0755), perm)
 		return nil
 	}
 	fakeOS.OpenFifoFn = func(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
@@ -211,11 +304,11 @@ func TestRunPodSandbox(t *testing.T) {
 	require.NotNil(t, res)
 	id := res.GetPodSandboxId()
 
-	assert.Len(t, dirs, 1)
+	sandboxRootDir := getSandboxRootDir(c.rootDir, id)
-	assert.Equal(t, getSandboxRootDir(c.rootDir, id), dirs[0], "sandbox root directory should be created")
+	assert.Contains(t, dirs[0], sandboxRootDir, "sandbox root directory should be created")
 
 	assert.Len(t, pipes, 2)
-	_, stdout, stderr := getStreamingPipes(getSandboxRootDir(c.rootDir, id))
+	_, stdout, stderr := getStreamingPipes(sandboxRootDir)
 	assert.Contains(t, pipes, stdout, "sandbox stdout pipe should be created")
 	assert.Contains(t, pipes, stderr, "sandbox stderr pipe should be created")