github/containerd
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Merge pull request #9787 from AkihiroSuda/cri-rro-kep-3857

Browse Source 
KEP-3857: Recursive Read-only (RRO) mounts
This commit is contained in:
Phil Estes
2024-02-21 18:52:43 +00:00
committed by GitHub
parent b8654e36f4 9077d13728
commit 8ce402c24c
16 changed files with 1292 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -74,7 +74,7 @@ require (
k8s.io/apimachinery v0.29.1
k8s.io/client-go v0.29.1
k8s.io/component-base v0.29.1
k8s.io/cri-api v0.29.1
k8s.io/cri-api v0.30.0-alpha.2.0.20240216190946-4e003cc3b0a4
k8s.io/klog/v2 v2.110.1
k8s.io/kubelet v0.29.1
k8s.io/utils v0.0.0-20230726121419-3b25d923346b

4
go.sum
View File

@@ -557,8 +557,8 @@ k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A=
k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks=
k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw=
k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc=
k8s.io/cri-api v0.29.1 h1:pQwYDahnAX9K8KtdV8PD1eeNexMJojEj1t/5kAMX61E=
k8s.io/cri-api v0.29.1/go.mod h1:9fQTFm+wi4FLyqrkVUoMJiUB3mE74XrVvHz8uFY/sSw=
k8s.io/cri-api v0.30.0-alpha.2.0.20240216190946-4e003cc3b0a4 h1:MkxF8QPcofA/nw9k03EQcMkCdP2RcyDZeF1Zda9m/3w=
k8s.io/cri-api v0.30.0-alpha.2.0.20240216190946-4e003cc3b0a4/go.mod h1:9fQTFm+wi4FLyqrkVUoMJiUB3mE74XrVvHz8uFY/sSw=
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=

21
internal/cri/opts/spec_linux_opts.go
View File

@@ -30,6 +30,7 @@ import (
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux/label"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
crierrors "k8s.io/cri-api/pkg/errors"
"github.com/containerd/containerd/v2/core/containers"
"github.com/containerd/containerd/v2/core/mount"
@@ -39,7 +40,7 @@ import (
)
// WithMounts sorts and adds runtime and CRI mounts to the spec
func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string) oci.SpecOpts {
func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*runtime.Mount, mountLabel string, handler *runtime.RuntimeHandler) oci.SpecOpts {
return func(ctx context.Context, client oci.Client, _ *containers.Container, s *runtimespec.Spec) (err error) {
// mergeMounts merge CRI mounts with extra mounts. If a mount destination
// is mounted by both a CRI mount and an extra mount, the CRI mount will
@@ -151,8 +152,24 @@ func WithMounts(osi osinterface.OS, config *runtime.ContainerConfig, extra []*ru
// NOTE(random-liu): we don't change all mounts to `ro` when root filesystem
// is readonly. This is different from docker's behavior, but make more sense.
if mount.GetReadonly() {
options = append(options, "ro")
if mount.GetRecursiveReadOnly() {
if handler == nil || !handler.Features.RecursiveReadOnlyMounts {
return fmt.Errorf("%w: runtime handler does not support recursive read-only mounts (hostPath=%q)",
crierrors.ErrRROUnsupported, mount.HostPath)
}
if mount.Propagation != runtime.MountPropagation_PROPAGATION_PRIVATE {
return fmt.Errorf("recursive read-only mount needs private propagation, got %q (hostPath=%q)",
mount.Propagation.String(), mount.HostPath)
}
options = append(options, "rro")
} else {
options = append(options, "ro")
}
} else {
if mount.GetRecursiveReadOnly() {
return fmt.Errorf("recursive read-only mount conflicts with RW mount (hostPath=%q)",
mount.HostPath)
}
options = append(options, "rw")
}

18
internal/cri/server/container_create.go
View File

@@ -167,6 +167,14 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
if err != nil {
return nil, fmt.Errorf("failed to get sandbox runtime: %w", err)
}
var runtimeHandler *runtime.RuntimeHandler
for _, f := range c.runtimeHandlers {
f := f
if f.Name == sandbox.Metadata.RuntimeHandler {
runtimeHandler = f
break
}
}
log.G(ctx).Debugf("Use OCI runtime %+v for sandbox %q and container %q", ociRuntime, sandboxID, id)
spec, err := c.buildContainerSpec(
@@ -182,6 +190,7 @@ func (c *criService) CreateContainer(ctx context.Context, r *runtime.CreateConta
&image.ImageSpec.Config,
volumeMounts,
ociRuntime,
runtimeHandler,
)
if err != nil {
return nil, fmt.Errorf("failed to generate container %q spec: %w", id, err)
@@ -530,6 +539,7 @@ func (c *criService) buildContainerSpec(
imageConfig *imagespec.ImageConfig,
extraMounts []*runtime.Mount,
ociRuntime criconfig.Runtime,
runtimeHandler *runtime.RuntimeHandler,
) (_ *runtimespec.Spec, retErr error) {
var (
specOpts []oci.SpecOpts
@@ -559,6 +569,7 @@ func (c *criService) buildContainerSpec(
imageConfig,
append(linuxMounts, extraMounts...),
ociRuntime,
runtimeHandler,
)
case isWindows:
specOpts, err = c.buildWindowsSpec(
@@ -573,6 +584,7 @@ func (c *criService) buildContainerSpec(
imageConfig,
extraMounts,
ociRuntime,
runtimeHandler,
)
case isDarwin:
specOpts, err = c.buildDarwinSpec(
@@ -585,6 +597,7 @@ func (c *criService) buildContainerSpec(
imageConfig,
extraMounts,
ociRuntime,
runtimeHandler,
)
default:
return nil, fmt.Errorf("unsupported spec platform: %s", platform.OS)
@@ -609,6 +622,7 @@ func (c *criService) buildLinuxSpec(
imageConfig *imagespec.ImageConfig,
extraMounts []*runtime.Mount,
ociRuntime criconfig.Runtime,
runtimeHandler *runtime.RuntimeHandler,
) (_ []oci.SpecOpts, retErr error) {
specOpts := []oci.SpecOpts{
oci.WithoutRunMount,
@@ -683,7 +697,7 @@ func (c *criService) buildLinuxSpec(
}
}()
specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel))
specOpts = append(specOpts, customopts.WithMounts(c.os, config, extraMounts, mountLabel, runtimeHandler))
if !c.config.DisableProcMount {
// Change the default masked/readonly paths to empty slices
@@ -841,6 +855,7 @@ func (c *criService) buildWindowsSpec(
imageConfig *imagespec.ImageConfig,
extraMounts []*runtime.Mount,
ociRuntime criconfig.Runtime,
runtimeHandler *runtime.RuntimeHandler,
) (_ []oci.SpecOpts, retErr error) {
var specOpts []oci.SpecOpts
specOpts = append(specOpts, customopts.WithProcessCommandLineOrArgsForWindows(config, imageConfig))
@@ -935,6 +950,7 @@ func (c *criService) buildDarwinSpec(
imageConfig *imagespec.ImageConfig,
extraMounts []*runtime.Mount,
ociRuntime criconfig.Runtime,
runtimeHandler *runtime.RuntimeHandler,
) (_ []oci.SpecOpts, retErr error) {
specOpts := []oci.SpecOpts{
customopts.WithProcessArgs(config, imageConfig),

38
internal/cri/server/container_create_linux_test.go
View File

@@ -255,7 +255,7 @@ func TestContainerCapabilities(t *testing.T) {
c.allCaps = allCaps
containerConfig.Linux.SecurityContext.Capabilities = test.capability
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
if selinux.GetEnabled() {
@@ -290,7 +290,7 @@ func TestContainerSpecTty(t *testing.T) {
c := newTestCRIService()
for _, tty := range []bool{true, false} {
containerConfig.Tty = tty
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
assert.Equal(t, tty, spec.Process.Terminal)
@@ -317,7 +317,7 @@ func TestContainerSpecDefaultPath(t *testing.T) {
imageConfig.Env = append(imageConfig.Env, pathenv)
expected = pathenv
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
assert.Contains(t, spec.Process.Env, expected)
@@ -334,7 +334,7 @@ func TestContainerSpecReadonlyRootfs(t *testing.T) {
c := newTestCRIService()
for _, readonly := range []bool{true, false} {
containerConfig.Linux.SecurityContext.ReadonlyRootfs = readonly
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
assert.Equal(t, readonly, spec.Root.Readonly)
@@ -368,7 +368,7 @@ func TestContainerSpecWithExtraMounts(t *testing.T) {
Readonly: false,
},
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, extraMounts, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
var mounts, sysMounts []runtimespec.Mount
@@ -435,7 +435,7 @@ func TestContainerAndSandboxPrivileged(t *testing.T) {
sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
Privileged: test.sandboxPrivileged,
}
_, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
_, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
if test.expectError {
assert.Error(t, err)
} else {
@@ -476,7 +476,7 @@ func TestPrivilegedBindMount(t *testing.T) {
containerConfig.Linux.SecurityContext.Privileged = test.privileged
sandboxConfig.Linux.SecurityContext.Privileged = test.privileged
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
if test.expectedSysFSRO {
@@ -597,7 +597,7 @@ func TestMountPropagation(t *testing.T) {
var spec runtimespec.Spec
spec.Linux = &runtimespec.Linux{}
err := opts.WithMounts(c.os, config, []*runtime.Mount{test.criMount}, "")(context.Background(), nil, nil, &spec)
err := opts.WithMounts(c.os, config, []*runtime.Mount{test.criMount}, "", nil)(context.Background(), nil, nil, &spec)
if test.expectErr {
require.Error(t, err)
} else {
@@ -648,7 +648,7 @@ func TestPidNamespace(t *testing.T) {
test := test
t.Run(test.desc, func(t *testing.T) {
containerConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{Pid: test.pidNS}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
assert.Contains(t, spec.Linux.Namespaces, test.expected)
})
@@ -823,7 +823,7 @@ func TestUserNamespace(t *testing.T) {
sandboxUserns = test.sandboxUserNS
}
sandboxConfig.Linux.SecurityContext.NamespaceOptions = &runtime.NamespaceOption{UsernsOptions: sandboxUserns}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
if test.err {
require.Error(t, err)
@@ -853,7 +853,7 @@ func TestNoDefaultRunMount(t *testing.T) {
ociRuntime := config.Runtime{}
c := newTestCRIService()
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
for _, mount := range spec.Mounts {
assert.NotEqual(t, "/run", mount.Destination)
@@ -1282,7 +1282,7 @@ func TestMaskedAndReadonlyPaths(t *testing.T) {
sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
Privileged: test.privileged,
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
if !test.privileged { // specCheck presumes an unprivileged container
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -1335,7 +1335,7 @@ func TestHostname(t *testing.T) {
sandboxConfig.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{Network: test.networkNs},
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
assert.Contains(t, spec.Process.Env, test.expectedEnv)
@@ -1348,7 +1348,7 @@ func TestDisableCgroup(t *testing.T) {
ociRuntime := config.Runtime{}
c := newTestCRIService()
c.config.DisableCgroup = true
spec, err := c.buildContainerSpec(currentPlatform, "test-id", "sandbox-id", 1234, "", "container-name", testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, "test-id", "sandbox-id", 1234, "", "container-name", testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
t.Log("resource limit should not be set")
@@ -1503,7 +1503,7 @@ additional-group-for-root:x:22222:root
containerConfig.Linux.SecurityContext = test.securityContext
imageConfig.User = test.imageConfigUser
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
spec.Root.Path = tempRootDir // simulating /etc/{passwd, group}
@@ -1579,7 +1579,7 @@ func TestNonRootUserAndDevices(t *testing.T) {
},
}
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil)
assert.NoError(t, err)
assert.Equal(t, test.expectedDeviceUID, *spec.Linux.Devices[0].UID)
@@ -1653,7 +1653,7 @@ func TestPrivilegedDevices(t *testing.T) {
PrivilegedWithoutHostDevices: test.privilegedWithoutHostDevices,
PrivilegedWithoutHostDevicesAllDevicesAllowed: test.privilegedWithoutHostDevicesAllDevicesAllowed,
}
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, t.Name(), testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
hostDevicesRaw, err := oci.HostDevices()
@@ -1708,7 +1708,7 @@ func TestBaseOCISpec(t *testing.T) {
testPid := uint32(1234)
containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -2040,7 +2040,7 @@ containerEdits:
},
} {
t.Run(test.description, func(t *testing.T) {
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)

6
internal/cri/server/container_create_test.go
View File

@@ -70,7 +70,7 @@ func TestGeneralContainerSpec(t *testing.T) {
c := newTestCRIService()
testSandboxID := "sandbox-id"
testContainerName := "container-name"
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
require.NoError(t, err)
specCheck(t, testID, testSandboxID, testPid, spec)
}
@@ -147,7 +147,7 @@ func TestPodAnnotationPassthroughContainerSpec(t *testing.T) {
PodAnnotations: test.podAnnotations,
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName,
containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -512,7 +512,7 @@ func TestContainerAnnotationPassthroughContainerSpec(t *testing.T) {
ContainerAnnotations: test.containerAnnotations,
}
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName,
containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, testSandboxID, testPid, spec)

10
internal/cri/server/container_create_windows_test.go
View File

@@ -157,7 +157,7 @@ func TestContainerWindowsNetworkNamespace(t *testing.T) {
c := newTestCRIService()
containerConfig, sandboxConfig, imageConfig, specCheck := getCreateContainerTestData()
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -179,7 +179,7 @@ func TestMountCleanPath(t *testing.T) {
ContainerPath: "c:/test/container-path",
HostPath: "c:/test/host-path",
})
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -199,7 +199,7 @@ func TestMountNamedPipe(t *testing.T) {
ContainerPath: `\\.\pipe\foo`,
HostPath: `\\.\pipe\foo`,
})
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
spec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, testSandboxID, testPid, spec)
@@ -251,7 +251,7 @@ func TestHostProcessRequirements(t *testing.T) {
sandboxConfig.Windows.SecurityContext = &runtime.WindowsSandboxSecurityContext{
HostProcess: test.sandboxHostProcess,
}
_, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime)
_, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, "", testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, ociRuntime, nil)
if test.expectError {
assert.Error(t, err)
} else {
@@ -348,7 +348,7 @@ func TestEntrypointAndCmdForArgsEscaped(t *testing.T) {
Args: test.args,
Windows: &runtime.WindowsContainerConfig{},
}
runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{})
runtimeSpec, err := c.buildContainerSpec(currentPlatform, testID, testSandboxID, testPid, nsPath, testContainerName, testImageName, containerConfig, sandboxConfig, imageConfig, nil, config.Runtime{}, nil)
assert.NoError(t, err)
assert.NotNil(t, runtimeSpec)

98
internal/cri/server/service.go
View File

@@ -21,18 +21,25 @@ import (
"fmt"
"io"
"net/http"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/containerd/go-cni"
"github.com/containerd/log"
"github.com/containerd/typeurl/v2"
"github.com/opencontainers/runtime-spec/specs-go/features"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"k8s.io/kubelet/pkg/cri/streaming"
introspectionapi "github.com/containerd/containerd/v2/api/services/introspection/v1"
apitypes "github.com/containerd/containerd/v2/api/types"
containerd "github.com/containerd/containerd/v2/client"
_ "github.com/containerd/containerd/v2/core/runtime" // for typeurl init
"github.com/containerd/containerd/v2/core/sandbox"
"github.com/containerd/containerd/v2/internal/cri/config"
criconfig "github.com/containerd/containerd/v2/internal/cri/config"
"github.com/containerd/containerd/v2/internal/cri/nri"
"github.com/containerd/containerd/v2/internal/cri/server/podsandbox"
@@ -46,8 +53,13 @@ import (
"github.com/containerd/containerd/v2/internal/registrar"
"github.com/containerd/containerd/v2/pkg/oci"
osinterface "github.com/containerd/containerd/v2/pkg/os"
"github.com/containerd/containerd/v2/plugins"
"github.com/containerd/containerd/v2/plugins/services/introspection"
"github.com/containerd/containerd/v2/protobuf"
)
var kernelSupportsRRO bool
// defaultNetworkPlugin is used for the default CNI configuration
const defaultNetworkPlugin = "default"
@@ -135,6 +147,8 @@ type criService struct {
nri *nri.API
// sandboxService is the sandbox related service for CRI
sandboxService sandboxService
// runtimeHandlers contains runtime handler info
runtimeHandlers []*runtime.RuntimeHandler
}
type CRIServiceOptions struct {
@@ -157,6 +171,7 @@ type CRIServiceOptions struct {
// NewCRIService returns a new instance of CRIService
func NewCRIService(options *CRIServiceOptions) (CRIService, runtime.RuntimeServiceServer, error) {
ctx := context.Background()
var err error
labels := label.NewStore()
config := options.RuntimeService.Config()
@@ -222,6 +237,11 @@ func NewCRIService(options *CRIServiceOptions) (CRIService, runtime.RuntimeServi
c.nri = options.NRI
c.runtimeHandlers, err = c.introspectRuntimeHandlers(ctx)
if err != nil {
return nil, nil, fmt.Errorf("failed to introspect runtime handlers: %w", err)
}
return c, c, nil
}
@@ -340,3 +360,81 @@ func (c *criService) Close() error {
func (c *criService) IsInitialized() bool {
return c.initialized.Load()
}
func (c *criService) introspectRuntimeHandlers(ctx context.Context) ([]*runtime.RuntimeHandler, error) {
var res []*runtime.RuntimeHandler
intro := c.client.IntrospectionService()
for name, r := range c.config.Runtimes {
h := runtime.RuntimeHandler{
Name: name,
}
rawFeatures, err := introspectRuntimeFeatures(ctx, intro, r)
if err != nil {
log.G(ctx).WithError(err).Debugf("failed to introspect features of runtime %q", name)
} else {
h.Features = &runtime.RuntimeHandlerFeatures{}
if slices.Contains(rawFeatures.MountOptions, "rro") {
if kernelSupportsRRO {
log.G(ctx).Debugf("runtime %q supports recursive read-only mounts", name)
h.Features.RecursiveReadOnlyMounts = true
} else {
log.G(ctx).Debugf("runtime %q supports recursive read-only mounts, but the kernel does not", name)
}
}
}
res = append(res, &h)
if name == c.config.DefaultRuntimeName {
defH := h
defH.Name = "" // denotes default
res = append(res, &defH)
}
}
return res, nil
}
func introspectRuntimeFeatures(ctx context.Context, intro introspection.Service, r config.Runtime) (*features.Features, error) {
if r.Type != plugins.RuntimeRuncV2 {
return nil, fmt.Errorf("introspecting OCI runtime features needs the runtime type to be %q, got %q",
plugins.RuntimeRuncV2, r.Type)
// For other runtimes, protobuf.MarshalAnyToProto will cause nil panic during typeurl dereference
}
infoReq := &introspectionapi.PluginInfoRequest{
Type: string(plugins.RuntimePluginV2), // "io.containerd.runtime.v2"
ID: "task",
}
rr := &apitypes.RuntimeRequest{
RuntimePath: r.Type, // "io.containerd.runc.v2"
}
if r.Path != "" {
rr.RuntimePath = r.Path // "/usr/local/bin/crun"
}
options, err := config.GenerateRuntimeOptions(r)
if err != nil {
return nil, err
}
rr.Options, err = protobuf.MarshalAnyToProto(options)
if err != nil {
return nil, fmt.Errorf("failed to marshal %T: %w", options, err)
}
infoReq.Options, err = protobuf.MarshalAnyToProto(rr)
if err != nil {
return nil, fmt.Errorf("failed to marshal %T: %w", rr, err)
}
infoResp, err := intro.PluginInfo(ctx, infoReq)
if err != nil {
return nil, fmt.Errorf("failed to call PluginInfo: %w", err)
}
var info apitypes.RuntimeInfo
if err := typeurl.UnmarshalTo(infoResp.Extra, &info); err != nil {
return nil, fmt.Errorf("failed to get runtime info from plugin info: %w", err)
}
featuresX, err := typeurl.UnmarshalAny(info.Features)
if err != nil {
return nil, fmt.Errorf("failed to unmarshal Features (%T): %w", info.Features, err)
}
features, ok := featuresX.(*features.Features)
if !ok {
return nil, fmt.Errorf("unknown features type %T", featuresX)
}
return features, nil
}

9
internal/cri/server/service_linux.go
View File

@@ -23,11 +23,20 @@ import (
"tags.cncf.io/container-device-interface/pkg/cdi"
"github.com/containerd/containerd/v2/pkg/cap"
"github.com/containerd/containerd/v2/pkg/kernelversion"
"github.com/containerd/containerd/v2/pkg/userns"
"github.com/containerd/go-cni"
"github.com/containerd/log"
)
func init() {
var err error
kernelSupportsRRO, err = kernelversion.GreaterEqualThan(kernelversion.KernelVersion{Kernel: 5, Major: 12})
if err != nil {
panic(fmt.Errorf("failed to check kernel version: %w", err))
}
}
// networkAttachCount is the minimum number of networks the PodSandbox
// attaches to
const networkAttachCount = 2

1
internal/cri/server/status.go
View File

@@ -58,6 +58,7 @@ func (c *criService) Status(ctx context.Context, r *runtime.StatusRequest) (*run
runtimeCondition,
networkCondition,
}},
RuntimeHandlers: c.runtimeHandlers,
}
if r.Verbose {
configByt, err := json.Marshal(c.config)

4
script/test/utils.sh
View File

@@ -348,4 +348,8 @@ readiness_check() {
echo "$attempt_num attempt \"$command\"! Trying again in $attempt_num seconds..."
sleep $(( attempt_num++ ))
done
set -x
cat "${report_dir}/containerd.log"
cat "${config_file}"
set +x
}

1476
vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.pb.go generated vendored
View File

File diff suppressed because it is too large Load Diff

27
vendor/k8s.io/cri-api/pkg/apis/runtime/v1/api.proto generated vendored
View File

@@ -235,6 +235,15 @@ message Mount {
repeated IDMapping uidMappings = 6;
// GidMappings specifies the runtime GID mappings for the mount.
repeated IDMapping gidMappings = 7;
// If set to true, the mount is made recursive read-only.
// In this CRI API, recursive_read_only is a plain true/false boolean, although its equivalent
// in the Kubernetes core API is a quaternary that can be nil, "Enabled", "IfPossible", or "Disabled".
// kubelet translates that quaternary value in the core API into a boolean in this CRI API.
// Remarks:
// - nil is just treated as false
// - when set to true, readonly must be explicitly set to true, and propagation must be PRIVATE (0).
// - (readonly == false && recursive_read_only == false) does not make the mount read-only.
bool recursive_read_only = 8;
}
// IDMapping describes host to container ID mappings for a pod sandbox.
@@ -1528,6 +1537,22 @@ message StatusRequest {
bool verbose = 1;
}
message RuntimeHandlerFeatures {
// recursive_read_only_mounts is set to true if the runtime handler supports
// recursive read-only mounts.
// For runc-compatible runtimes, availability of this feature can be detected by checking whether
// the Linux kernel version is >= 5.12, and, `runc features | jq .mountOptions` contains "rro".
bool recursive_read_only_mounts = 1;
}
message RuntimeHandler {
// Name must be unique in StatusResponse.
// An empty string denotes the default handler.
string name = 1;
// Supported features.
RuntimeHandlerFeatures features = 2;
}
message StatusResponse {
// Status of the Runtime.
RuntimeStatus status = 1;
@@ -1536,6 +1561,8 @@ message StatusResponse {
// debug, e.g. plugins used by the container runtime.
// It should only be returned non-empty when Verbose is true.
map<string, string> info = 2;
// Runtime handlers.
repeated RuntimeHandler runtime_handlers = 3;
}
message ImageFsInfoRequest {}

19
vendor/k8s.io/cri-api/pkg/errors/doc.go generated vendored Normal file
View File

@@ -0,0 +1,19 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package errors provides helper functions for use by the kubelet
// to deal with CRI errors.
package errors // import "k8s.io/cri-api/pkg/errors"

51
vendor/k8s.io/cri-api/pkg/errors/errors.go generated vendored Normal file
View File

@@ -0,0 +1,51 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package errors
import (
"errors"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
var (
// ErrRegistryUnavailable - Get http error on the PullImage RPC call.
ErrRegistryUnavailable = errors.New("RegistryUnavailable")
// ErrSignatureValidationFailed - Unable to validate the image signature on the PullImage RPC call.
ErrSignatureValidationFailed = errors.New("SignatureValidationFailed")
// ErrRROUnsupported - Unable to enforce recursive readonly mounts
ErrRROUnsupported = errors.New("RROUnsupported")
)
// IsNotFound returns a boolean indicating whether the error
// is grpc not found error.
// See https://github.com/grpc/grpc/blob/master/doc/statuscodes.md
// for a list of grpc status codes.
func IsNotFound(err error) bool {
s, ok := status.FromError(err)
if !ok {
return ok
}
if s.Code() == codes.NotFound {
return true
}
return false
}

3
vendor/modules.txt vendored
View File

@@ -746,9 +746,10 @@ k8s.io/client-go/util/workqueue
# k8s.io/component-base v0.29.1
## explicit; go 1.21
k8s.io/component-base/logs/logreduction
# k8s.io/cri-api v0.29.1
# k8s.io/cri-api v0.30.0-alpha.2.0.20240216190946-4e003cc3b0a4
## explicit; go 1.21
k8s.io/cri-api/pkg/apis/runtime/v1
k8s.io/cri-api/pkg/errors
# k8s.io/klog/v2 v2.110.1
## explicit; go 1.13
k8s.io/klog/v2