Merge pull request #8327 from mxpv/linux-mounts

Keep linux mounts for linux sandboxes on Windows/Darwin
This commit is contained in:
Akihiro Suda 2023-03-30 17:21:21 +09:00 committed by GitHub
commit 812111eb08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 275 additions and 297 deletions

View File

@ -156,9 +156,6 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
log.G(ctx).Debugf("Ignoring volumes defined in image %v because IgnoreImageDefinedVolumes is set", image.ID)
}
// Generate container mounts.
mounts := c.containerMounts(sandboxID, config)
ociRuntime, err := c.getSandboxRuntime(sandboxConfig, sandbox.Metadata.RuntimeHandler)
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
@ -181,7 +178,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
config,
sandboxConfig,
&image.ImageSpec.Config,
append(mounts, volumeMounts...),
volumeMounts,
ociRuntime,
)
if err != nil {
@ -434,6 +431,10 @@ func (c *criService) buildContainerSpec(
switch {
case isLinux:
// Generate container mounts.
// No mounts are passed for other platforms.
linuxMounts := c.linuxContainerMounts(sandboxID, config)
specOpts, err = c.buildLinuxSpec(
id,
sandboxID,
@ -444,7 +445,7 @@ func (c *criService) buildContainerSpec(
config,
sandboxConfig,
imageConfig,
extraMounts,
append(linuxMounts, extraMounts...),
ociRuntime,
)
case isWindows:
@ -863,3 +864,60 @@ func (c *criService) buildDarwinSpec(
return specOpts, nil
}
// containerMounts sets up necessary container system file mounts
// including /dev/shm, /etc/hosts and /etc/resolv.conf.
func (c *criService) linuxContainerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
var mounts []*runtime.Mount
securityContext := config.GetLinux().GetSecurityContext()
if !isInCRIMounts(etcHostname, config.GetMounts()) {
// /etc/hostname is added since 1.1.6, 1.2.4 and 1.3.
// For in-place upgrade, the old sandbox doesn't have the hostname file,
// do not mount this in that case.
// TODO(random-liu): Remove the check and always mount this when
// containerd 1.1 and 1.2 are deprecated.
hostpath := c.getSandboxHostname(sandboxID)
if _, err := c.os.Stat(hostpath); err == nil {
mounts = append(mounts, &runtime.Mount{
ContainerPath: etcHostname,
HostPath: hostpath,
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
}
if !isInCRIMounts(etcHosts, config.GetMounts()) {
mounts = append(mounts, &runtime.Mount{
ContainerPath: etcHosts,
HostPath: c.getSandboxHosts(sandboxID),
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
// Mount sandbox resolv.config.
// TODO: Need to figure out whether we should always mount it as read-only
if !isInCRIMounts(resolvConfPath, config.GetMounts()) {
mounts = append(mounts, &runtime.Mount{
ContainerPath: resolvConfPath,
HostPath: c.getResolvPath(sandboxID),
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
if !isInCRIMounts(devShm, config.GetMounts()) {
sandboxDevShm := c.getSandboxDevShm(sandboxID)
if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
sandboxDevShm = devShm
}
mounts = append(mounts, &runtime.Mount{
ContainerPath: devShm,
HostPath: sandboxDevShm,
Readonly: false,
SelinuxRelabel: sandboxDevShm != devShm,
})
}
return mounts
}

View File

@ -51,63 +51,6 @@ const (
seccompDefaultProfile = dockerDefault
)
// containerMounts sets up necessary container system file mounts
// including /dev/shm, /etc/hosts and /etc/resolv.conf.
func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
var mounts []*runtime.Mount
securityContext := config.GetLinux().GetSecurityContext()
if !isInCRIMounts(etcHostname, config.GetMounts()) {
// /etc/hostname is added since 1.1.6, 1.2.4 and 1.3.
// For in-place upgrade, the old sandbox doesn't have the hostname file,
// do not mount this in that case.
// TODO(random-liu): Remove the check and always mount this when
// containerd 1.1 and 1.2 are deprecated.
hostpath := c.getSandboxHostname(sandboxID)
if _, err := c.os.Stat(hostpath); err == nil {
mounts = append(mounts, &runtime.Mount{
ContainerPath: etcHostname,
HostPath: hostpath,
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
}
if !isInCRIMounts(etcHosts, config.GetMounts()) {
mounts = append(mounts, &runtime.Mount{
ContainerPath: etcHosts,
HostPath: c.getSandboxHosts(sandboxID),
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
// Mount sandbox resolv.config.
// TODO: Need to figure out whether we should always mount it as read-only
if !isInCRIMounts(resolvConfPath, config.GetMounts()) {
mounts = append(mounts, &runtime.Mount{
ContainerPath: resolvConfPath,
HostPath: c.getResolvPath(sandboxID),
Readonly: securityContext.GetReadonlyRootfs(),
SelinuxRelabel: true,
})
}
if !isInCRIMounts(devShm, config.GetMounts()) {
sandboxDevShm := c.getSandboxDevShm(sandboxID)
if securityContext.GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
sandboxDevShm = devShm
}
mounts = append(mounts, &runtime.Mount{
ContainerPath: devShm,
HostPath: sandboxDevShm,
Readonly: false,
SelinuxRelabel: sandboxDevShm != devShm,
})
}
return mounts
}
func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
var specOpts []oci.SpecOpts
securityContext := config.GetLinux().GetSecurityContext()

View File

@ -18,7 +18,6 @@ package sbserver
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
@ -360,23 +359,16 @@ func TestContainerSpecWithExtraMounts(t *testing.T) {
HostPath: "test-sys-extra",
Readonly: false,
},
{
ContainerPath: "/dev",
HostPath: "test-dev-extra",
Readonly: false,
},
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
var mounts, sysMounts, devMounts []runtimespec.Mount
var mounts, sysMounts []runtimespec.Mount
for _, m := range spec.Mounts {
if strings.HasPrefix(m.Destination, "test-container-path") {
mounts = append(mounts, m)
} else if m.Destination == "/sys" {
sysMounts = append(sysMounts, m)
} else if strings.HasPrefix(m.Destination, "/dev") {
devMounts = append(devMounts, m)
}
}
t.Logf("CRI mount should override extra mount")
@ -388,11 +380,6 @@ func TestContainerSpecWithExtraMounts(t *testing.T) {
require.Len(t, sysMounts, 1)
assert.Equal(t, "test-sys-extra", sysMounts[0].Source)
assert.Contains(t, sysMounts[0].Options, "rw")
t.Logf("Dev mount should override all default dev mounts")
require.Len(t, devMounts, 1)
assert.Equal(t, "test-dev-extra", devMounts[0].Source)
assert.Contains(t, devMounts[0].Options, "rw")
}
func TestContainerAndSandboxPrivileged(t *testing.T) {
@ -444,173 +431,6 @@ func TestContainerAndSandboxPrivileged(t *testing.T) {
}
}
func TestContainerMounts(t *testing.T) {
const testSandboxID = "test-id"
for desc, test := range map[string]struct {
statFn func(string) (os.FileInfo, error)
criMounts []*runtime.Mount
securityContext *runtime.LinuxContainerSecurityContext
expectedMounts []*runtime.Mount
}{
"should setup ro mount when rootfs is read-only": {
securityContext: &runtime.LinuxContainerSecurityContext{
ReadonlyRootfs: true,
},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
"should setup rw mount when rootfs is read-write": {
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
"should use host /dev/shm when host ipc is set": {
securityContext: &runtime.LinuxContainerSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{Ipc: runtime.NamespaceMode_NODE},
},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: "/dev/shm",
Readonly: false,
},
},
},
"should skip container mounts if already mounted by CRI": {
criMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: "/test-etc-hostname",
},
{
ContainerPath: "/etc/hosts",
HostPath: "/test-etc-host",
},
{
ContainerPath: resolvConfPath,
HostPath: "test-resolv-conf",
},
{
ContainerPath: "/dev/shm",
HostPath: "test-dev-shm",
},
},
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: nil,
},
"should skip hostname mount if the old sandbox doesn't have hostname file": {
statFn: func(path string) (os.FileInfo, error) {
assert.Equal(t, filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"), path)
return nil, errors.New("random error")
},
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
} {
t.Run(desc, func(t *testing.T) {
config := &runtime.ContainerConfig{
Metadata: &runtime.ContainerMetadata{
Name: "test-name",
Attempt: 1,
},
Mounts: test.criMounts,
Linux: &runtime.LinuxContainerConfig{
SecurityContext: test.securityContext,
},
}
c := newTestCRIService()
c.os.(*ostesting.FakeOS).StatFn = test.statFn
mounts := c.containerMounts(testSandboxID, config)
assert.Equal(t, test.expectedMounts, mounts, desc)
})
}
}
func TestPrivilegedBindMount(t *testing.T) {
testPid := uint32(1234)
c := newTestCRIService()

View File

@ -26,12 +26,6 @@ import (
"github.com/containerd/containerd/snapshots"
)
// containerMounts sets up necessary container system file mounts
// including /dev/shm, /etc/hosts and /etc/resolv.conf.
func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
return []*runtime.Mount{}
}
func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
return []oci.SpecOpts{}, nil
}

View File

@ -18,10 +18,13 @@ package sbserver
import (
"context"
"errors"
"os"
"path/filepath"
goruntime "runtime"
"testing"
ostesting "github.com/containerd/containerd/pkg/os/testing"
"github.com/containerd/containerd/platforms"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
@ -435,3 +438,170 @@ func TestBaseRuntimeSpec(t *testing.T) {
assert.Equal(t, filepath.Join("/", constants.K8sContainerdNamespace, "id1"), out.Linux.CgroupsPath)
}
func TestLinuxContainerMounts(t *testing.T) {
const testSandboxID = "test-id"
for desc, test := range map[string]struct {
statFn func(string) (os.FileInfo, error)
criMounts []*runtime.Mount
securityContext *runtime.LinuxContainerSecurityContext
expectedMounts []*runtime.Mount
}{
"should setup ro mount when rootfs is read-only": {
securityContext: &runtime.LinuxContainerSecurityContext{
ReadonlyRootfs: true,
},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: true,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
"should setup rw mount when rootfs is read-write": {
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
"should use host /dev/shm when host ipc is set": {
securityContext: &runtime.LinuxContainerSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{Ipc: runtime.NamespaceMode_NODE},
},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: "/dev/shm",
Readonly: false,
},
},
},
"should skip container mounts if already mounted by CRI": {
criMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hostname",
HostPath: "/test-etc-hostname",
},
{
ContainerPath: "/etc/hosts",
HostPath: "/test-etc-host",
},
{
ContainerPath: resolvConfPath,
HostPath: "test-resolv-conf",
},
{
ContainerPath: "/dev/shm",
HostPath: "test-dev-shm",
},
},
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: nil,
},
"should skip hostname mount if the old sandbox doesn't have hostname file": {
statFn: func(path string) (os.FileInfo, error) {
assert.Equal(t, filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hostname"), path)
return nil, errors.New("random error")
},
securityContext: &runtime.LinuxContainerSecurityContext{},
expectedMounts: []*runtime.Mount{
{
ContainerPath: "/etc/hosts",
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "hosts"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: resolvConfPath,
HostPath: filepath.Join(testRootDir, sandboxesDir, testSandboxID, "resolv.conf"),
Readonly: false,
SelinuxRelabel: true,
},
{
ContainerPath: "/dev/shm",
HostPath: filepath.Join(testStateDir, sandboxesDir, testSandboxID, "shm"),
Readonly: false,
SelinuxRelabel: true,
},
},
},
} {
t.Run(desc, func(t *testing.T) {
config := &runtime.ContainerConfig{
Metadata: &runtime.ContainerMetadata{
Name: "test-name",
Attempt: 1,
},
Mounts: test.criMounts,
Linux: &runtime.LinuxContainerConfig{
SecurityContext: test.securityContext,
},
}
c := newTestCRIService()
c.os.(*ostesting.FakeOS).StatFn = test.statFn
mounts := c.linuxContainerMounts(testSandboxID, config)
assert.Equal(t, test.expectedMounts, mounts, desc)
})
}
}

View File

@ -26,11 +26,6 @@ import (
"github.com/containerd/containerd/snapshots"
)
// No container mounts for windows.
func (c *criService) containerMounts(sandboxID string, config *runtime.ContainerConfig) []*runtime.Mount {
return nil
}
// No extra spec options needed for windows.
func (c *criService) containerSpecOpts(config *runtime.ContainerConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
return nil, nil

View File

@ -92,8 +92,49 @@ const (
// runtimeRunhcsV1 is the runtime type for runhcs.
runtimeRunhcsV1 = "io.containerd.runhcs.v1"
// devShm is the default path of /dev/shm.
devShm = "/dev/shm"
// etcHosts is the default path of /etc/hosts file.
etcHosts = "/etc/hosts"
// etcHostname is the default path of /etc/hostname file.
etcHostname = "/etc/hostname"
// resolvConfPath is the abs path of resolv.conf on host or container.
resolvConfPath = "/etc/resolv.conf"
)
// getSandboxRootDir returns the root directory for managing sandbox files,
// e.g. hosts files.
func (c *criService) getSandboxRootDir(id string) string {
return filepath.Join(c.config.RootDir, sandboxesDir, id)
}
// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
// e.g. named pipes.
func (c *criService) getVolatileSandboxRootDir(id string) string {
return filepath.Join(c.config.StateDir, sandboxesDir, id)
}
// getSandboxHostname returns the hostname file path inside the sandbox root directory.
func (c *criService) getSandboxHostname(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "hostname")
}
// getSandboxHosts returns the hosts file path inside the sandbox root directory.
func (c *criService) getSandboxHosts(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "hosts")
}
// getResolvPath returns resolv.conf filepath for specified sandbox.
func (c *criService) getResolvPath(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
}
// getSandboxDevShm returns the shm file path inside the sandbox root directory.
func (c *criService) getSandboxDevShm(id string) string {
return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
}
// makeSandboxName generates sandbox name from sandbox metadata. The name
// generated is unique as long as sandbox metadata is unique.
func makeSandboxName(s *runtime.PodSandboxMetadata) string {

View File

@ -38,49 +38,6 @@ import (
"github.com/containerd/containerd/pkg/seutil"
)
const (
// devShm is the default path of /dev/shm.
devShm = "/dev/shm"
// etcHosts is the default path of /etc/hosts file.
etcHosts = "/etc/hosts"
// etcHostname is the default path of /etc/hostname file.
etcHostname = "/etc/hostname"
// resolvConfPath is the abs path of resolv.conf on host or container.
resolvConfPath = "/etc/resolv.conf"
)
// getSandboxRootDir returns the root directory for managing sandbox files,
// e.g. hosts files.
func (c *criService) getSandboxRootDir(id string) string {
return filepath.Join(c.config.RootDir, sandboxesDir, id)
}
// getVolatileSandboxRootDir returns the root directory for managing volatile sandbox files,
// e.g. named pipes.
func (c *criService) getVolatileSandboxRootDir(id string) string {
return filepath.Join(c.config.StateDir, sandboxesDir, id)
}
// getSandboxHostname returns the hostname file path inside the sandbox root directory.
func (c *criService) getSandboxHostname(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "hostname")
}
// getSandboxHosts returns the hosts file path inside the sandbox root directory.
func (c *criService) getSandboxHosts(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "hosts")
}
// getResolvPath returns resolv.conf filepath for specified sandbox.
func (c *criService) getResolvPath(id string) string {
return filepath.Join(c.getSandboxRootDir(id), "resolv.conf")
}
// getSandboxDevShm returns the shm file path inside the sandbox root directory.
func (c *criService) getSandboxDevShm(id string) string {
return filepath.Join(c.getVolatileSandboxRootDir(id), "shm")
}
// apparmorEnabled returns true if apparmor is enabled, supported by the host,
// if apparmor_parser is installed, and if we are not running docker-in-docker.
func (c *criService) apparmorEnabled() bool {