github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Add sandbox /dev/shm.

Browse Source 
Signed-off-by: Lantao Liu <lantaol@google.com>
This commit is contained in:
Lantao Liu 2017-06-07 02:28:53 +00:00
parent 5398a3b7ec
commit 9d5990fe4f
6 changed files with 196 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/os/testing/fake_os.go
View File

@ -183,6 +183,7 @@ func (f *FakeOS) WriteFile(filename string, data []byte, perm os.FileMode) error
if f.WriteFileFn != nil {
return f.WriteFileFn(filename, data, perm)
}
return nil
}
// Mount is a fake call that invokes MountFn or just return nil.

12
pkg/server/container_start.go
View File

@ -315,7 +315,15 @@ func (c *criContainerdService) generateContainerMounts(sandboxRootDir string, co
Readonly: securityContext.GetReadonlyRootfs(),
})
// TODO(random-liu): [P0] Mount sandbox /dev/shm.
sandboxDevShm := getSandboxDevShm(sandboxRootDir)
if securityContext.GetNamespaceOptions().GetHostIpc() {
sandboxDevShm = devShm
}
mounts = append(mounts, &runtime.Mount{
ContainerPath: devShm,
HostPath: sandboxDevShm,
Readonly: false,
})
return mounts
}
@ -416,6 +424,8 @@ func addOCIDevices(g *generate.Generator, devs []*runtime.Device, privileged boo
}
// addOCIBindMounts adds bind mounts.
// TODO(random-liu): Figure out whether we need to change all CRI mounts to readonly when
// rootfs is readonly. (https://github.com/moby/moby/blob/master/daemon/oci_linux.go)
func addOCIBindMounts(g *generate.Generator, mounts []*runtime.Mount, privileged bool) {
for _, mount := range mounts {
dst := mount.GetContainerPath()

34
pkg/server/container_start_test.go
View File

@ -324,6 +324,11 @@ func TestGenerateContainerMounts(t *testing.T) {
HostPath: testSandboxRootDir + "/resolv.conf",
Readonly: true,
},
{
ContainerPath: "/dev/shm",
HostPath: testSandboxRootDir + "/shm",
Readonly: false,
},
},
},
"should setup rw mount when rootfs is read-write": {
@ -336,7 +341,34 @@ func TestGenerateContainerMounts(t *testing.T) {
},
{
ContainerPath: resolvConfPath,
HostPath: getResolvPath(testSandboxRootDir),
HostPath: testSandboxRootDir + "/resolv.conf",
Readonly: false,
},
{
ContainerPath: "/dev/shm",
HostPath: testSandboxRootDir + "/shm",
Readonly: false,
},
},
},
"should use host /dev/shm when host ipc is set": {
securityContext: &runtime.LinuxContainerSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{HostIpc: true},
},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hosts",
HostPath: testSandboxRootDir + "/hosts",
Readonly: false,
},
{
ContainerPath: resolvConfPath,
HostPath: testSandboxRootDir + "/resolv.conf",
Readonly: false,
},
{
ContainerPath: "/dev/shm",
HostPath: "/dev/shm",
Readonly: false,
},
},

9
pkg/server/helpers.go
View File

@ -58,6 +58,8 @@ const (
// defaultSandboxImage is the image used by sandbox container.
// TODO(random-liu): [P1] Build schema 2 pause image and use it here.
defaultSandboxImage = "gcr.io/google.com/noogler-kubernetes/pause-amd64:3.0"
// defaultShmSize is the default size of the sandbox shm.
defaultShmSize = int64(1024 * 1024 * 64)
// relativeRootfsPath is the rootfs path relative to bundle path.
relativeRootfsPath = "rootfs"
// defaultRuntime is the runtime to use in containerd. We may support
@ -88,6 +90,8 @@ const (
utsNSFormat = "/proc/%v/ns/uts"
// pidNSFormat is the format of pid namespace of a process.
pidNSFormat = "/proc/%v/ns/pid"
// devShm is the default path of /dev/shm.
devShm = "/dev/shm"
// etcHosts is the default path of /etc/hosts file.
etcHosts = "/etc/hosts"
// resolvConfPath is the abs path of resolv.conf on host or container.
@ -159,6 +163,11 @@ func getResolvPath(sandboxRoot string) string {
return filepath.Join(sandboxRoot, "resolv.conf")
}
// getSandboxDevShm returns the shm file path inside the sandbox root directory.
func getSandboxDevShm(sandboxRootDir string) string {
return filepath.Join(sandboxRootDir, "shm")
}
// prepareStreamingPipes prepares stream named pipe for container. returns nil
// streaming handler if corresponding stream path is empty.
func (c *criContainerdService) prepareStreamingPipes(ctx context.Context, stdin, stdout, stderr string) (

39
pkg/server/sandbox_run.go
View File

@ -19,6 +19,7 @@ package server
import (
"encoding/json"
"fmt"
"os"
"strings"
"time"
@ -31,6 +32,7 @@ import (
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/runtime-tools/generate"
"golang.org/x/net/context"
"golang.org/x/sys/unix"
runtime "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
"github.com/kubernetes-incubator/cri-containerd/pkg/metadata"
@ -137,8 +139,11 @@ func (c *criContainerdService) RunPodSandbox(ctx context.Context, r *runtime.Run
if err = c.setupSandboxFiles(sandboxRootDir, config); err != nil {
return nil, fmt.Errorf("failed to setup sandbox files: %v", err)
}
// No need to cleanup on error, because the whole sandbox root directory will be removed
// on error.
defer func() {
if retErr != nil {
c.cleanupSandboxFiles(sandboxRootDir, config)
}
}()
// Start sandbox container.
spec, err := c.generateSandboxContainerSpec(id, config, imageMeta.Config)
@ -301,7 +306,7 @@ func (c *criContainerdService) generateSandboxContainerSpec(id string, config *r
func (c *criContainerdService) setupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) error {
// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
sandboxEtcHosts := getSandboxHosts(rootDir)
if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0666); err != nil {
if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
return fmt.Errorf("failed to generate sandbox hosts file %q: %v", sandboxEtcHosts, err)
}
@ -328,8 +333,22 @@ func (c *criContainerdService) setupSandboxFiles(rootDir string, config *runtime
}
}
// TODO(random-liu): [P0] Deal with /dev/shm. Use host for HostIpc, and create and mount for
// non-HostIpc. What about mqueue?
// Setup sandbox /dev/shm.
if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostIpc() {
if _, err := c.os.Stat(devShm); err != nil {
return fmt.Errorf("host %q is not available for host ipc: %v", devShm, err)
}
} else {
sandboxDevShm := getSandboxDevShm(rootDir)
if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
return fmt.Errorf("failed to create sandbox shm: %v", err)
}
shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
if err := c.os.Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
return fmt.Errorf("failed to mount sandbox shm: %v", err)
}
}
return nil
}
@ -356,3 +375,13 @@ func parseDNSOptions(servers, searches, options []string) (string, error) {
return resolvContent, nil
}
// cleanupSandboxFiles only unmount files, we rely on the removal of sandbox root directory to remove files.
// Each cleanup task should log error instead of returning, so as to keep on cleanup on error.
func (c *criContainerdService) cleanupSandboxFiles(rootDir string, config *runtime.PodSandboxConfig) {
if !config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetHostIpc() {
if err := c.os.Unmount(getSandboxDevShm(rootDir), unix.MNT_DETACH); err != nil && os.IsNotExist(err) {
glog.Errorf("failed to unmount sandbox shm: %v", err)
}
}
}

121
pkg/server/sandbox_run_test.go
View File

@ -159,18 +159,112 @@ func TestGenerateSandboxContainerSpec(t *testing.T) {
func TestSetupSandboxFiles(t *testing.T) {
testRootDir := "test-sandbox-root"
expectedCopys := [][]interface{}{
{"/etc/hosts", testRootDir + "/hosts", os.FileMode(0666)},
{"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644)},
}
for desc, test := range map[string]struct {
dnsConfig *runtime.DNSConfig
hostIpc bool
expectedCalls []ostesting.CalledDetail
}{
"should check host /dev/shm existence when hostIpc is true": {
hostIpc: true,
expectedCalls: []ostesting.CalledDetail{
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
},
},
{
Name: "Stat",
Arguments: []interface{}{"/dev/shm"},
},
},
},
"should create new /etc/resolv.conf if DNSOptions is set": {
dnsConfig: &runtime.DNSConfig{
Servers: []string{"8.8.8.8"},
Searches: []string{"114.114.114.114"},
Options: []string{"timeout:1"},
},
hostIpc: true,
expectedCalls: []ostesting.CalledDetail{
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
},
},
{
Name: "WriteFile",
Arguments: []interface{}{
testRootDir + "/resolv.conf", []byte(`search 114.114.114.114
nameserver 8.8.8.8
options timeout:1
`), os.FileMode(0644),
},
},
{
Name: "Stat",
Arguments: []interface{}{"/dev/shm"},
},
},
},
"should create sandbox shm when hostIpc is false": {
hostIpc: false,
expectedCalls: []ostesting.CalledDetail{
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts", testRootDir + "/hosts", os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/resolv.conf", testRootDir + "/resolv.conf", os.FileMode(0644),
},
},
{
Name: "MkdirAll",
Arguments: []interface{}{
testRootDir + "/shm", os.FileMode(0700),
},
},
{
Name: "Mount",
// Ignore arguments which are too complex to check.
},
},
},
} {
t.Logf("TestCase %q", desc)
c := newTestCRIContainerdService()
var copys [][]interface{}
c.os.(*ostesting.FakeOS).CopyFileFn = func(src string, dest string, perm os.FileMode) error {
copys = append(copys, []interface{}{src, dest, perm})
return nil
cfg := &runtime.PodSandboxConfig{
DnsConfig: test.dnsConfig,
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{
HostIpc: test.hostIpc,
},
},
},
}
c.setupSandboxFiles(testRootDir, cfg)
calls := c.os.(*ostesting.FakeOS).GetCalls()
assert.Len(t, calls, len(test.expectedCalls))
for i, expected := range test.expectedCalls {
if expected.Arguments == nil {
// Ignore arguments.
expected.Arguments = calls[i].Arguments
}
assert.Equal(t, expected, calls[i])
}
}
c.setupSandboxFiles(testRootDir, nil)
assert.Equal(t, expectedCopys, copys, "should copy expected files for sandbox")
}
func TestRunPodSandbox(t *testing.T) {
@ -184,7 +278,6 @@ func TestRunPodSandbox(t *testing.T) {
var pipes []string
fakeOS.MkdirAllFn = func(path string, perm os.FileMode) error {
dirs = append(dirs, path)
assert.Equal(t, os.FileMode(0755), perm)
return nil
}
fakeOS.OpenFifoFn = func(ctx context.Context, fn string, flag int, perm os.FileMode) (io.ReadWriteCloser, error) {
@ -211,11 +304,11 @@ func TestRunPodSandbox(t *testing.T) {
require.NotNil(t, res)
id := res.GetPodSandboxId()
assert.Len(t, dirs, 1)
assert.Equal(t, getSandboxRootDir(c.rootDir, id), dirs[0], "sandbox root directory should be created")
sandboxRootDir := getSandboxRootDir(c.rootDir, id)
assert.Contains(t, dirs[0], sandboxRootDir, "sandbox root directory should be created")
assert.Len(t, pipes, 2)
_, stdout, stderr := getStreamingPipes(getSandboxRootDir(c.rootDir, id))
_, stdout, stderr := getStreamingPipes(sandboxRootDir)
assert.Contains(t, pipes, stdout, "sandbox stdout pipe should be created")
assert.Contains(t, pipes, stderr, "sandbox stderr pipe should be created")