github/containerd
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

Cleanup CRI files

Browse Source 
Signed-off-by: Maksym Pavlenko <pavlenko.maksym@gmail.com>
This commit is contained in:
Maksym Pavlenko 2022-07-28 14:01:50 -07:00
parent c085fac1e5
commit 3d028308ef
15 changed files with 2 additions and 1969 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/cri/sbserver/helpers_linux.go
View File

@ -35,16 +35,11 @@ import (
"github.com/containerd/containerd/pkg/seutil" "github.com/containerd/containerd/pkg/seutil"
"github.com/moby/sys/mountinfo" "github.com/moby/sys/mountinfo"
"github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux/label"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
) )
const ( const (
// defaultSandboxOOMAdj is default omm adj for sandbox container. (kubernetes#47938).
defaultSandboxOOMAdj = -998
// defaultShmSize is the default size of the sandbox shm.
defaultShmSize = int64(1024 * 1024 * 64)
// relativeRootfsPath is the rootfs path relative to bundle path. // relativeRootfsPath is the rootfs path relative to bundle path.
relativeRootfsPath = "rootfs" relativeRootfsPath = "rootfs"
// devShm is the default path of /dev/shm. // devShm is the default path of /dev/shm.
@ -115,14 +110,6 @@ func toLabel(selinuxOptions *runtime.SELinuxOption) ([]string, error) {
return labels, nil return labels, nil
} }
func initLabelsFromOpt(selinuxOpts *runtime.SELinuxOption) (string, string, error) {
labels, err := toLabel(selinuxOpts)
if err != nil {
return "", "", err
}
return label.InitLabels(labels)
}
func checkSelinuxLevel(level string) error { func checkSelinuxLevel(level string) error {
if len(level) == 0 { if len(level) == 0 {
return nil return nil

100
pkg/cri/sbserver/helpers_linux_test.go
View File

@ -1,100 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sbserver
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/assert"
"golang.org/x/sys/unix"
)
func TestGetCgroupsPath(t *testing.T) {
testID := "test-id"
for desc, test := range map[string]struct {
cgroupsParent string
expected string
}{
"should support regular cgroup path": {
cgroupsParent: "/a/b",
expected: "/a/b/test-id",
},
"should support systemd cgroup path": {
cgroupsParent: "/a.slice/b.slice",
expected: "b.slice:cri-containerd:test-id",
},
"should support tailing slash for regular cgroup path": {
cgroupsParent: "/a/b/",
expected: "/a/b/test-id",
},
"should support tailing slash for systemd cgroup path": {
cgroupsParent: "/a.slice/b.slice/",
expected: "b.slice:cri-containerd:test-id",
},
"should treat root cgroup as regular cgroup path": {
cgroupsParent: "/",
expected: "/test-id",
},
} {
t.Run(desc, func(t *testing.T) {
got := getCgroupsPath(test.cgroupsParent, testID)
assert.Equal(t, test.expected, got)
})
}
}
func TestEnsureRemoveAllWithMount(t *testing.T) {
if os.Getuid() != 0 {
t.Skip("skipping test that requires root")
}
var err error
dir1 := t.TempDir()
dir2 := t.TempDir()
bindDir := filepath.Join(dir1, "bind")
if err := os.MkdirAll(bindDir, 0755); err != nil {
t.Fatal(err)
}
if err := unix.Mount(dir2, bindDir, "none", unix.MS_BIND, ""); err != nil {
t.Fatal(err)
}
done := make(chan struct{})
go func() {
err = ensureRemoveAll(context.Background(), dir1)
close(done)
}()
select {
case <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(5 * time.Second):
t.Fatal("timeout waiting for EnsureRemoveAll to finish")
}
if _, err := os.Stat(dir1); !os.IsNotExist(err) {
t.Fatalf("expected %q to not exist", dir1)
}
}

157
pkg/cri/sbserver/helpers_selinux_linux_test.go
View File

@ -1,157 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sbserver
import (
"testing"
"github.com/opencontainers/selinux/go-selinux"
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)
func TestInitSelinuxOpts(t *testing.T) {
if !selinux.GetEnabled() {
t.Skip("selinux is not enabled")
}
for desc, test := range map[string]struct {
selinuxOpt *runtime.SELinuxOption
processLabel string
mountLabel string
expectErr bool
}{
"Should return empty strings for processLabel and mountLabel when selinuxOpt is nil": {
selinuxOpt: nil,
processLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}",
mountLabel: ".*:c[0-9]{1,3},c[0-9]{1,3}",
},
"Should overlay fields on processLabel when selinuxOpt has been initialized partially": {
selinuxOpt: &runtime.SELinuxOption{
User: "",
Role: "user_r",
Type: "",
Level: "s0:c1,c2",
},
processLabel: "system_u:user_r:(container_file_t|svirt_lxc_net_t):s0:c1,c2",
mountLabel: "system_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
},
"Should be resolved correctly when selinuxOpt has been initialized completely": {
selinuxOpt: &runtime.SELinuxOption{
User: "user_u",
Role: "user_r",
Type: "user_t",
Level: "s0:c1,c2",
},
processLabel: "user_u:user_r:user_t:s0:c1,c2",
mountLabel: "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0:c1,c2",
},
"Should be resolved correctly when selinuxOpt has been initialized with level=''": {
selinuxOpt: &runtime.SELinuxOption{
User: "user_u",
Role: "user_r",
Type: "user_t",
Level: "",
},
processLabel: "user_u:user_r:user_t:s0:c[0-9]{1,3},c[0-9]{1,3}",
mountLabel: "user_u:object_r:(container_file_t|svirt_sandbox_file_t):s0",
},
"Should return error when the format of 'level' is not correct": {
selinuxOpt: &runtime.SELinuxOption{
User: "user_u",
Role: "user_r",
Type: "user_t",
Level: "s0,c1,c2",
},
expectErr: true,
},
} {
t.Run(desc, func(t *testing.T) {
processLabel, mountLabel, err := initLabelsFromOpt(test.selinuxOpt)
if test.expectErr {
assert.Error(t, err)
} else {
assert.Regexp(t, test.processLabel, processLabel)
assert.Regexp(t, test.mountLabel, mountLabel)
}
})
}
}
func TestCheckSelinuxLevel(t *testing.T) {
for desc, test := range map[string]struct {
level string
expectNoMatch bool
}{
"s0": {
level: "s0",
},
"s0-s0": {
level: "s0-s0",
},
"s0:c0": {
level: "s0:c0",
},
"s0:c0.c3": {
level: "s0:c0.c3",
},
"s0:c0,c3": {
level: "s0:c0,c3",
},
"s0-s0:c0,c3": {
level: "s0-s0:c0,c3",
},
"s0-s0:c0,c3.c6": {
level: "s0-s0:c0,c3.c6",
},
"s0-s0:c0,c3.c6,c8.c10": {
level: "s0-s0:c0,c3.c6,c8.c10",
},
"s0-s0:c0,c3.c6,c8,c10": {
level: "s0-s0:c0,c3.c6",
},
"s0,c0,c3": {
level: "s0,c0,c3",
expectNoMatch: true,
},
"s0:c0.c3.c6": {
level: "s0:c0.c3.c6",
expectNoMatch: true,
},
"s0-s0,c0,c3": {
level: "s0-s0,c0,c3",
expectNoMatch: true,
},
"s0-s0:c0.c3.c6": {
level: "s0-s0:c0.c3.c6",
expectNoMatch: true,
},
"s0-s0:c0,c3.c6.c8": {
level: "s0-s0:c0,c3.c6.c8",
expectNoMatch: true,
},
} {
t.Run(desc, func(t *testing.T) {
err := checkSelinuxLevel(test.level)
if test.expectNoMatch {
assert.Error(t, err)
} else {
assert.NoError(t, err)
}
})
}
}

125
pkg/cri/sbserver/podsandbox/helpers.go
View File

@ -30,9 +30,7 @@ import (
"github.com/containerd/containerd/log" "github.com/containerd/containerd/log"
"github.com/containerd/containerd/oci" "github.com/containerd/containerd/oci"
criconfig "github.com/containerd/containerd/pkg/cri/config" criconfig "github.com/containerd/containerd/pkg/cri/config"
containerstore "github.com/containerd/containerd/pkg/cri/store/container"
imagestore "github.com/containerd/containerd/pkg/cri/store/image" imagestore "github.com/containerd/containerd/pkg/cri/store/image"
sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
ctrdutil "github.com/containerd/containerd/pkg/cri/util" ctrdutil "github.com/containerd/containerd/pkg/cri/util"
runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1" runtimeoptions "github.com/containerd/containerd/pkg/runtimeoptions/v1"
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
@ -46,80 +44,26 @@ import (
runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
imagedigest "github.com/opencontainers/go-digest" imagedigest "github.com/opencontainers/go-digest"
"github.com/pelletier/go-toml" "github.com/pelletier/go-toml"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
) )
const ( const (
// errorStartReason is the exit reason when fails to start container.
errorStartReason = "StartError"
// errorStartExitCode is the exit code when fails to start container.
// 128 is the same with Docker's behavior.
// TODO(windows): Figure out what should be used for windows.
errorStartExitCode = 128
// completeExitReason is the exit reason when container exits with code 0.
completeExitReason = "Completed"
// errorExitReason is the exit reason when container exits with code non-zero.
errorExitReason = "Error"
// oomExitReason is the exit reason when process in container is oom killed.
oomExitReason = "OOMKilled"
// sandboxesDir contains all sandbox root. A sandbox root is the running // sandboxesDir contains all sandbox root. A sandbox root is the running
// directory of the sandbox, all files created for the sandbox will be // directory of the sandbox, all files created for the sandbox will be
// placed under this directory. // placed under this directory.
sandboxesDir = "sandboxes" sandboxesDir = "sandboxes"
// containersDir contains all container root.
containersDir = "containers"
// Delimiter used to construct container/sandbox names.
nameDelimiter = "_"
// criContainerdPrefix is common prefix for cri-containerd // criContainerdPrefix is common prefix for cri-containerd
criContainerdPrefix = "io.cri-containerd" criContainerdPrefix = "io.cri-containerd"
// containerKindLabel is a label key indicating container is sandbox container or application container // containerKindLabel is a label key indicating container is sandbox container or application container
containerKindLabel = criContainerdPrefix + ".kind" containerKindLabel = criContainerdPrefix + ".kind"
// containerKindSandbox is a label value indicating container is sandbox container // containerKindSandbox is a label value indicating container is sandbox container
containerKindSandbox = "sandbox" containerKindSandbox = "sandbox"
// containerKindContainer is a label value indicating container is application container
containerKindContainer = "container"
// imageLabelKey is the label key indicating the image is managed by cri plugin.
imageLabelKey = criContainerdPrefix + ".image"
// imageLabelValue is the label value indicating the image is managed by cri plugin.
imageLabelValue = "managed"
// sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest // sandboxMetadataExtension is an extension name that identify metadata of sandbox in CreateContainerRequest
sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata" sandboxMetadataExtension = criContainerdPrefix + ".sandbox.metadata"
// containerMetadataExtension is an extension name that identify metadata of container in CreateContainerRequest
containerMetadataExtension = criContainerdPrefix + ".container.metadata"
// defaultIfName is the default network interface for the pods
defaultIfName = "eth0"
// runtimeRunhcsV1 is the runtime type for runhcs. // runtimeRunhcsV1 is the runtime type for runhcs.
runtimeRunhcsV1 = "io.containerd.runhcs.v1" runtimeRunhcsV1 = "io.containerd.runhcs.v1"
) )
// makeSandboxName generates sandbox name from sandbox metadata. The name
// generated is unique as long as sandbox metadata is unique.
func makeSandboxName(s *runtime.PodSandboxMetadata) string {
return strings.Join([]string{
s.Name, // 0
s.Namespace, // 1
s.Uid, // 2
fmt.Sprintf("%d", s.Attempt), // 3
}, nameDelimiter)
}
// makeContainerName generates container name from sandbox and container metadata.
// The name generated is unique as long as the sandbox container combination is
// unique.
func makeContainerName(c *runtime.ContainerMetadata, s *runtime.PodSandboxMetadata) string {
return strings.Join([]string{
c.Name, // 0: container name
s.Name, // 1: pod name
s.Namespace, // 2: pod namespace
s.Uid, // 3: pod uid
fmt.Sprintf("%d", c.Attempt), // 4: attempt number of creating the container
}, nameDelimiter)
}
// getSandboxRootDir returns the root directory for managing sandbox files, // getSandboxRootDir returns the root directory for managing sandbox files,
// e.g. hosts files. // e.g. hosts files.
func (c *Controller) getSandboxRootDir(id string) string { func (c *Controller) getSandboxRootDir(id string) string {
@ -132,23 +76,6 @@ func (c *Controller) getVolatileSandboxRootDir(id string) string {
return filepath.Join(c.config.StateDir, sandboxesDir, id) return filepath.Join(c.config.StateDir, sandboxesDir, id)
} }
// getContainerRootDir returns the root directory for managing container files,
// e.g. state checkpoint.
func (c *Controller) getContainerRootDir(id string) string {
return filepath.Join(c.config.RootDir, containersDir, id)
}
// getVolatileContainerRootDir returns the root directory for managing volatile container files,
// e.g. named pipes.
func (c *Controller) getVolatileContainerRootDir(id string) string {
return filepath.Join(c.config.StateDir, containersDir, id)
}
// criContainerStateToString formats CRI container state to string.
func criContainerStateToString(state runtime.ContainerState) string {
return runtime.ContainerState_name[int32(state)]
}
// getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference. // getRepoDigestAngTag returns image repoDigest and repoTag of the named image reference.
func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) { func getRepoDigestAndTag(namedRef docker.Named, digest imagedigest.Digest, schema1 bool) (string, string) {
var repoTag, repoDigest string var repoTag, repoDigest string
@ -192,22 +119,6 @@ func getUserFromImage(user string) (*int64, string) {
return &uid, "" return &uid, ""
} }
// isInCRIMounts checks whether a destination is in CRI mount list.
func isInCRIMounts(dst string, mounts []*runtime.Mount) bool {
for _, m := range mounts {
if filepath.Clean(m.ContainerPath) == filepath.Clean(dst) {
return true
}
}
return false
}
// filterLabel returns a label filter. Use `%q` here because containerd
// filter needs extra quote to work properly.
func filterLabel(k, v string) string {
return fmt.Sprintf("labels.%q==%q", k, v)
}
// buildLabel builds the labels from config to be passed to containerd // buildLabel builds the labels from config to be passed to containerd
func buildLabels(configLabels, imageConfigLabels map[string]string, containerType string) map[string]string { func buildLabels(configLabels, imageConfigLabels map[string]string, containerType string) map[string]string {
labels := make(map[string]string) labels := make(map[string]string)
@ -229,16 +140,6 @@ func buildLabels(configLabels, imageConfigLabels map[string]string, containerTyp
return labels return labels
} }
// toRuntimeAuthConfig converts cri plugin auth config to runtime auth config.
func toRuntimeAuthConfig(a criconfig.AuthConfig) *runtime.AuthConfig {
return &runtime.AuthConfig{
Username: a.Username,
Password: a.Password,
Auth: a.Auth,
IdentityToken: a.IdentityToken,
}
}
// parseImageReferences parses a list of arbitrary image references and returns // parseImageReferences parses a list of arbitrary image references and returns
// the repotags and repodigests // the repotags and repodigests
func parseImageReferences(refs []string) ([]string, []string) { func parseImageReferences(refs []string) ([]string, []string) {
@ -310,32 +211,6 @@ func getRuntimeOptions(c containers.Container) (interface{}, error) {
return opts, nil return opts, nil
} }
const (
// unknownExitCode is the exit code when exit reason is unknown.
unknownExitCode = 255
// unknownExitReason is the exit reason when exit reason is unknown.
unknownExitReason = "Unknown"
)
// unknownContainerStatus returns the default container status when its status is unknown.
func unknownContainerStatus() containerstore.Status {
return containerstore.Status{
CreatedAt: 0,
StartedAt: 0,
FinishedAt: 0,
ExitCode: unknownExitCode,
Reason: unknownExitReason,
Unknown: true,
}
}
// unknownSandboxStatus returns the default sandbox status when its status is unknown.
func unknownSandboxStatus() sandboxstore.Status {
return sandboxstore.Status{
State: sandboxstore.StateUnknown,
}
}
// getPassthroughAnnotations filters requested pod annotations by comparing // getPassthroughAnnotations filters requested pod annotations by comparing
// against permitted annotations for the given runtime. // against permitted annotations for the given runtime.
func getPassthroughAnnotations(podAnnotations map[string]string, func getPassthroughAnnotations(podAnnotations map[string]string,

22
pkg/cri/sbserver/podsandbox/helpers_linux.go
View File

@ -30,7 +30,6 @@ import (
"github.com/containerd/containerd/log" "github.com/containerd/containerd/log"
"github.com/containerd/containerd/mount" "github.com/containerd/containerd/mount"
"github.com/containerd/containerd/pkg/apparmor"
"github.com/containerd/containerd/pkg/seccomp" "github.com/containerd/containerd/pkg/seccomp"
"github.com/containerd/containerd/pkg/seutil" "github.com/containerd/containerd/pkg/seutil"
"github.com/moby/sys/mountinfo" "github.com/moby/sys/mountinfo"
@ -51,12 +50,8 @@ const (
devShm = "/dev/shm" devShm = "/dev/shm"
// etcHosts is the default path of /etc/hosts file. // etcHosts is the default path of /etc/hosts file.
etcHosts = "/etc/hosts" etcHosts = "/etc/hosts"
// etcHostname is the default path of /etc/hostname file.
etcHostname = "/etc/hostname"
// resolvConfPath is the abs path of resolv.conf on host or container. // resolvConfPath is the abs path of resolv.conf on host or container.
resolvConfPath = "/etc/resolv.conf" resolvConfPath = "/etc/resolv.conf"
// hostnameEnv is the key for HOSTNAME env.
hostnameEnv = "HOSTNAME"
) )
// getCgroupsPath generates container cgroups path. // getCgroupsPath generates container cgroups path.
@ -138,27 +133,10 @@ func checkSelinuxLevel(level string) error {
return nil return nil
} }
// apparmorEnabled returns true if apparmor is enabled, supported by the host,
// if apparmor_parser is installed, and if we are not running docker-in-docker.
func (c *Controller) apparmorEnabled() bool {
if c.config.DisableApparmor {
return false
}
return apparmor.HostSupports()
}
func (c *Controller) seccompEnabled() bool { func (c *Controller) seccompEnabled() bool {
return seccomp.IsEnabled() return seccomp.IsEnabled()
} }
// openLogFile opens/creates a container log file.
func openLogFile(path string) (*os.File, error) {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return nil, err
}
return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
}
// unmountRecursive unmounts the target and all mounts underneath, starting with // unmountRecursive unmounts the target and all mounts underneath, starting with
// the deepest mount first. // the deepest mount first.
func unmountRecursive(ctx context.Context, target string) error { func unmountRecursive(ctx context.Context, target string) error {

5
pkg/cri/sbserver/podsandbox/helpers_other.go
View File

@ -26,11 +26,6 @@ import (
"github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-spec/specs-go"
) )
// openLogFile opens/creates a container log file.
func openLogFile(path string) (*os.File, error) {
return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
}
// ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can // ensureRemoveAll wraps `os.RemoveAll` to check for specific errors that can
// often be remedied. // often be remedied.
// Only use `ensureRemoveAll` if you really want to make every effort to remove // Only use `ensureRemoveAll` if you really want to make every effort to remove

135
pkg/cri/sbserver/podsandbox/helpers_windows.go
View File

@ -19,145 +19,10 @@ package podsandbox
import ( import (
"context" "context"
"os" "os"
"path/filepath"
"syscall"
"github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-spec/specs-go"
) )
// openLogFile opens/creates a container log file.
// It specifies `FILE_SHARE_DELETE` option to make sure
// log files can be rotated by kubelet.
// TODO(windows): Use golang support after 1.14. (https://github.com/golang/go/issues/32088)
func openLogFile(path string) (*os.File, error) {
path = fixLongPath(path)
if len(path) == 0 {
return nil, syscall.ERROR_FILE_NOT_FOUND
}
pathp, err := syscall.UTF16PtrFromString(path)
if err != nil {
return nil, err
}
createmode := uint32(syscall.OPEN_ALWAYS)
access := uint32(syscall.FILE_APPEND_DATA)
sharemode := uint32(syscall.FILE_SHARE_READ | syscall.FILE_SHARE_WRITE | syscall.FILE_SHARE_DELETE)
h, err := syscall.CreateFile(pathp, access, sharemode, nil, createmode, syscall.FILE_ATTRIBUTE_NORMAL, 0)
if err != nil {
return nil, err
}
return os.NewFile(uintptr(h), path), nil
}
// Copyright (c) 2009 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
//
// This is copied from https://golang.org/src/path/filepath/path_windows.go.
func fixLongPath(path string) string {
// Do nothing (and don't allocate) if the path is "short".
// Empirically (at least on the Windows Server 2013 builder),
// the kernel is arbitrarily okay with < 248 bytes. That
// matches what the docs above say:
// "When using an API to create a directory, the specified
// path cannot be so long that you cannot append an 8.3 file
// name (that is, the directory name cannot exceed MAX_PATH
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
//
// The MSDN docs appear to say that a normal path that is 248 bytes long
// will work; empirically the path must be less then 248 bytes long.
if len(path) < 248 {
// Don't fix. (This is how Go 1.7 and earlier worked,
// not automatically generating the \\?\ form)
return path
}
// The extended form begins with \\?\, as in
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
// The extended form disables evaluation of . and .. path
// elements and disables the interpretation of / as equivalent
// to \. The conversion here rewrites / to \ and elides
// . elements as well as trailing or duplicate separators. For
// simplicity it avoids the conversion entirely for relative
// paths or paths containing .. elements. For now,
// \\server\share paths are not converted to
// \\?\UNC\server\share paths because the rules for doing so
// are less well-specified.
if len(path) >= 2 && path[:2] == `\\` {
// Don't canonicalize UNC paths.
return path
}
if !filepath.IsAbs(path) {
// Relative path
return path
}
const prefix = `\\?`
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
copy(pathbuf, prefix)
n := len(path)
r, w := 0, len(prefix)
for r < n {
switch {
case os.IsPathSeparator(path[r]):
// empty block
r++
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
// /./
r++
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
// /../ is currently unhandled
return path
default:
pathbuf[w] = '\\'
w++
for ; r < n && !os.IsPathSeparator(path[r]); r++ {
pathbuf[w] = path[r]
w++
}
}
}
// A drive's root directory needs a trailing \
if w == len(`\\?\c:`) {
pathbuf[w] = '\\'
w++
}
return string(pathbuf[:w])
}
// ensureRemoveAll is a wrapper for os.RemoveAll on Windows. // ensureRemoveAll is a wrapper for os.RemoveAll on Windows.
func ensureRemoveAll(_ context.Context, dir string) error { func ensureRemoveAll(_ context.Context, dir string) error {
return os.RemoveAll(dir) return os.RemoveAll(dir)

4
pkg/cri/sbserver/podsandbox/sandbox_run.go
View File

@ -84,10 +84,10 @@ func (c *Controller) Start(ctx context.Context, id string) (_ uint32, retErr err
} }
log.G(ctx).WithField("podsandboxid", id).Debugf("sandbox container spec: %#+v", spew.NewFormatter(spec)) log.G(ctx).WithField("podsandboxid", id).Debugf("sandbox container spec: %#+v", spew.NewFormatter(spec))
processLabel := spec.Process.SelinuxLabel metadata.ProcessLabel = spec.Process.SelinuxLabel
defer func() { defer func() {
if retErr != nil { if retErr != nil {
selinux.ReleaseLabel(processLabel) selinux.ReleaseLabel(metadata.ProcessLabel)
} }
}() }()

291
pkg/cri/sbserver/sandbox_run_linux.go
View File

@ -19,305 +19,14 @@ package sbserver
import ( import (
"fmt" "fmt"
"os" "os"
"strconv"
"strings"
"github.com/containerd/containerd" "github.com/containerd/containerd"
"github.com/containerd/containerd/oci"
"github.com/containerd/containerd/plugin" "github.com/containerd/containerd/plugin"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux"
"golang.org/x/sys/unix"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
customopts "github.com/containerd/containerd/pkg/cri/opts"
osinterface "github.com/containerd/containerd/pkg/os" osinterface "github.com/containerd/containerd/pkg/os"
"github.com/containerd/containerd/pkg/userns"
) )
func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
// Creates a spec Generator with the default spec.
// TODO(random-liu): [P1] Compare the default settings with docker and containerd default.
specOpts := []oci.SpecOpts{
oci.WithoutRunMount,
customopts.WithoutDefaultSecuritySettings,
customopts.WithRelativeRoot(relativeRootfsPath),
oci.WithEnv(imageConfig.Env),
oci.WithRootFSReadonly(),
oci.WithHostname(config.GetHostname()),
}
if imageConfig.WorkingDir != "" {
specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
}
if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
// Pause image must have entrypoint or cmd.
return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
}
specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
// Set cgroups parent.
if c.config.DisableCgroup {
specOpts = append(specOpts, customopts.WithDisabledCgroups)
} else {
if config.GetLinux().GetCgroupParent() != "" {
cgroupsPath := getCgroupsPath(config.GetLinux().GetCgroupParent(), id)
specOpts = append(specOpts, oci.WithCgroup(cgroupsPath))
}
}
// When cgroup parent is not set, containerd-shim will create container in a child cgroup
// of the cgroup itself is in.
// TODO(random-liu): [P2] Set default cgroup path if cgroup parent is not specified.
// Set namespace options.
var (
securityContext = config.GetLinux().GetSecurityContext()
nsOptions = securityContext.GetNamespaceOptions()
)
if nsOptions.GetNetwork() == runtime.NamespaceMode_NODE {
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.NetworkNamespace))
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.UTSNamespace))
} else {
specOpts = append(specOpts, oci.WithLinuxNamespace(
runtimespec.LinuxNamespace{
Type: runtimespec.NetworkNamespace,
Path: nsPath,
}))
}
if nsOptions.GetPid() == runtime.NamespaceMode_NODE {
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.PIDNamespace))
}
if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
specOpts = append(specOpts, customopts.WithoutNamespace(runtimespec.IPCNamespace))
}
// It's fine to generate the spec before the sandbox /dev/shm
// is actually created.
sandboxDevShm := c.getSandboxDevShm(id)
if nsOptions.GetIpc() == runtime.NamespaceMode_NODE {
sandboxDevShm = devShm
}
// Remove the default /dev/shm mount from defaultMounts, it is added in oci/mounts.go.
specOpts = append(specOpts, oci.WithoutMounts(devShm))
// In future the when user-namespace is enabled, the `nosuid, nodev, noexec` flags are
// required, otherwise the remount will fail with EPERM. Just use them unconditionally,
// they are nice to have anyways.
specOpts = append(specOpts, oci.WithMounts([]runtimespec.Mount{
{
Source: sandboxDevShm,
Destination: devShm,
Type: "bind",
Options: []string{"rbind", "ro", "nosuid", "nodev", "noexec"},
},
// Add resolv.conf for katacontainers to setup the DNS of pod VM properly.
{
Source: c.getResolvPath(id),
Destination: resolvConfPath,
Type: "bind",
Options: []string{"rbind", "ro"},
},
}))
processLabel, mountLabel, err := initLabelsFromOpt(securityContext.GetSelinuxOptions())
if err != nil {
return nil, fmt.Errorf("failed to init selinux options %+v: %w", securityContext.GetSelinuxOptions(), err)
}
defer func() {
if retErr != nil {
selinux.ReleaseLabel(processLabel)
}
}()
supplementalGroups := securityContext.GetSupplementalGroups()
specOpts = append(specOpts,
customopts.WithSelinuxLabels(processLabel, mountLabel),
customopts.WithSupplementalGroups(supplementalGroups),
)
// Add sysctls
sysctls := config.GetLinux().GetSysctls()
if sysctls == nil {
sysctls = make(map[string]string)
}
_, ipUnprivilegedPortStart := sysctls["net.ipv4.ip_unprivileged_port_start"]
_, pingGroupRange := sysctls["net.ipv4.ping_group_range"]
if nsOptions.GetNetwork() != runtime.NamespaceMode_NODE {
if c.config.EnableUnprivilegedPorts && !ipUnprivilegedPortStart {
sysctls["net.ipv4.ip_unprivileged_port_start"] = "0"
}
if c.config.EnableUnprivilegedICMP && !pingGroupRange && !userns.RunningInUserNS() {
sysctls["net.ipv4.ping_group_range"] = "0 2147483647"
}
}
specOpts = append(specOpts, customopts.WithSysctls(sysctls))
// Note: LinuxSandboxSecurityContext does not currently provide an apparmor profile
if !c.config.DisableCgroup {
specOpts = append(specOpts, customopts.WithDefaultSandboxShares)
}
if res := config.GetLinux().GetResources(); res != nil {
specOpts = append(specOpts,
customopts.WithAnnotation(annotations.SandboxCPUPeriod, strconv.FormatInt(res.CpuPeriod, 10)),
customopts.WithAnnotation(annotations.SandboxCPUQuota, strconv.FormatInt(res.CpuQuota, 10)),
customopts.WithAnnotation(annotations.SandboxCPUShares, strconv.FormatInt(res.CpuShares, 10)),
customopts.WithAnnotation(annotations.SandboxMem, strconv.FormatInt(res.MemoryLimitInBytes, 10)))
}
specOpts = append(specOpts, customopts.WithPodOOMScoreAdj(int(defaultSandboxOOMAdj), c.config.RestrictOOMScoreAdj))
for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
runtimePodAnnotations) {
specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
}
specOpts = append(specOpts,
customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
customopts.WithAnnotation(annotations.SandboxID, id),
customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
)
return c.runtimeSpec(id, "", specOpts...)
}
// sandboxContainerSpecOpts generates OCI spec options for
// the sandbox container.
func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
var (
securityContext = config.GetLinux().GetSecurityContext()
specOpts []oci.SpecOpts
err error
)
ssp := securityContext.GetSeccomp()
if ssp == nil {
ssp, err = generateSeccompSecurityProfile(
securityContext.GetSeccompProfilePath(), //nolint:staticcheck // Deprecated but we don't want to remove yet
c.config.UnsetSeccompProfile)
if err != nil {
return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
}
}
seccompSpecOpts, err := c.generateSeccompSpecOpts(
ssp,
securityContext.GetPrivileged(),
c.seccompEnabled())
if err != nil {
return nil, fmt.Errorf("failed to generate seccomp spec opts: %w", err)
}
if seccompSpecOpts != nil {
specOpts = append(specOpts, seccompSpecOpts)
}
userstr, err := generateUserString(
"",
securityContext.GetRunAsUser(),
securityContext.GetRunAsGroup(),
)
if err != nil {
return nil, fmt.Errorf("failed to generate user string: %w", err)
}
if userstr == "" {
// Lastly, since no user override was passed via CRI try to set via OCI
// Image
userstr = imageConfig.User
}
if userstr != "" {
specOpts = append(specOpts, oci.WithUser(userstr))
}
return specOpts, nil
}
// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
// /etc/resolv.conf and /etc/hostname.
func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
sandboxEtcHostname := c.getSandboxHostname(id)
hostname := config.GetHostname()
if hostname == "" {
var err error
hostname, err = c.os.Hostname()
if err != nil {
return fmt.Errorf("failed to get hostname: %w", err)
}
}
if err := c.os.WriteFile(sandboxEtcHostname, []byte(hostname+"\n"), 0644); err != nil {
return fmt.Errorf("failed to write hostname to %q: %w", sandboxEtcHostname, err)
}
// TODO(random-liu): Consider whether we should maintain /etc/hosts and /etc/resolv.conf in kubelet.
sandboxEtcHosts := c.getSandboxHosts(id)
if err := c.os.CopyFile(etcHosts, sandboxEtcHosts, 0644); err != nil {
return fmt.Errorf("failed to generate sandbox hosts file %q: %w", sandboxEtcHosts, err)
}
// Set DNS options. Maintain a resolv.conf for the sandbox.
var err error
resolvContent := ""
if dnsConfig := config.GetDnsConfig(); dnsConfig != nil {
resolvContent, err = parseDNSOptions(dnsConfig.Servers, dnsConfig.Searches, dnsConfig.Options)
if err != nil {
return fmt.Errorf("failed to parse sandbox DNSConfig %+v: %w", dnsConfig, err)
}
}
resolvPath := c.getResolvPath(id)
if resolvContent == "" {
// copy host's resolv.conf to resolvPath
err = c.os.CopyFile(resolvConfPath, resolvPath, 0644)
if err != nil {
return fmt.Errorf("failed to copy host's resolv.conf to %q: %w", resolvPath, err)
}
} else {
err = c.os.WriteFile(resolvPath, []byte(resolvContent), 0644)
if err != nil {
return fmt.Errorf("failed to write resolv content to %q: %w", resolvPath, err)
}
}
// Setup sandbox /dev/shm.
if config.GetLinux().GetSecurityContext().GetNamespaceOptions().GetIpc() == runtime.NamespaceMode_NODE {
if _, err := c.os.Stat(devShm); err != nil {
return fmt.Errorf("host %q is not available for host ipc: %w", devShm, err)
}
} else {
sandboxDevShm := c.getSandboxDevShm(id)
if err := c.os.MkdirAll(sandboxDevShm, 0700); err != nil {
return fmt.Errorf("failed to create sandbox shm: %w", err)
}
shmproperty := fmt.Sprintf("mode=1777,size=%d", defaultShmSize)
if err := c.os.(osinterface.UNIX).Mount("shm", sandboxDevShm, "tmpfs", uintptr(unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV), shmproperty); err != nil {
return fmt.Errorf("failed to mount sandbox shm: %w", err)
}
}
return nil
}
// parseDNSOptions parse DNS options into resolv.conf format content,
// if none option is specified, will return empty with no error.
func parseDNSOptions(servers, searches, options []string) (string, error) {
resolvContent := ""
if len(searches) > 0 {
resolvContent += fmt.Sprintf("search %s\n", strings.Join(searches, " "))
}
if len(servers) > 0 {
resolvContent += fmt.Sprintf("nameserver %s\n", strings.Join(servers, "\nnameserver "))
}
if len(options) > 0 {
resolvContent += fmt.Sprintf("options %s\n", strings.Join(options, " "))
}
return resolvContent, nil
}
// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
// remove these files. Unmount should *NOT* return error if the mount point is already unmounted. // remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {

526
pkg/cri/sbserver/sandbox_run_linux_test.go
View File

@ -1,526 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sbserver
import (
"os"
"path/filepath"
"strconv"
"testing"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/opencontainers/selinux/go-selinux"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
v1 "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
"github.com/containerd/containerd/pkg/cri/opts"
ostesting "github.com/containerd/containerd/pkg/os/testing"
)
func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
config := &runtime.PodSandboxConfig{
Metadata: &runtime.PodSandboxMetadata{
Name: "test-name",
Uid: "test-uid",
Namespace: "test-ns",
Attempt: 1,
},
Hostname: "test-hostname",
LogDirectory: "test-log-directory",
Labels: map[string]string{"a": "b"},
Annotations: map[string]string{"c": "d"},
Linux: &runtime.LinuxPodSandboxConfig{
CgroupParent: "/test/cgroup/parent",
},
}
imageConfig := &imagespec.ImageConfig{
Env: []string{"a=b", "c=d"},
Entrypoint: []string{"/pause"},
Cmd: []string{"forever"},
WorkingDir: "/workspace",
}
specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
assert.Equal(t, "test-hostname", spec.Hostname)
assert.Equal(t, getCgroupsPath("/test/cgroup/parent", id), spec.Linux.CgroupsPath)
assert.Equal(t, relativeRootfsPath, spec.Root.Path)
assert.Equal(t, true, spec.Root.Readonly)
assert.Contains(t, spec.Process.Env, "a=b", "c=d")
assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args)
assert.Equal(t, "/workspace", spec.Process.Cwd)
assert.EqualValues(t, *spec.Linux.Resources.CPU.Shares, opts.DefaultSandboxCPUshares)
assert.EqualValues(t, *spec.Process.OOMScoreAdj, defaultSandboxOOMAdj)
t.Logf("Check PodSandbox annotations")
assert.Contains(t, spec.Annotations, annotations.SandboxID)
assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id)
assert.Contains(t, spec.Annotations, annotations.ContainerType)
assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox)
assert.Contains(t, spec.Annotations, annotations.SandboxNamespace)
assert.EqualValues(t, spec.Annotations[annotations.SandboxNamespace], "test-ns")
assert.Contains(t, spec.Annotations, annotations.SandboxName)
assert.EqualValues(t, spec.Annotations[annotations.SandboxName], "test-name")
assert.Contains(t, spec.Annotations, annotations.SandboxLogDir)
assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory")
if selinux.GetEnabled() {
assert.NotEqual(t, "", spec.Process.SelinuxLabel)
assert.NotEqual(t, "", spec.Linux.MountLabel)
}
}
return config, imageConfig, specCheck
}
func TestLinuxSandboxContainerSpec(t *testing.T) {
testID := "test-id"
nsPath := "test-cni"
for desc, test := range map[string]struct {
configChange func(*runtime.PodSandboxConfig)
specCheck func(*testing.T, *runtimespec.Spec)
expectErr bool
}{
"spec should reflect original config": {
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
// runtime spec should have expected namespaces enabled by default.
require.NotNil(t, spec.Linux)
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.NetworkNamespace,
Path: nsPath,
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.UTSNamespace,
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.PIDNamespace,
})
assert.Contains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.IPCNamespace,
})
assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "0")
assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647")
},
},
"host namespace": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
}
},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
// runtime spec should disable expected namespaces in host mode.
require.NotNil(t, spec.Linux)
assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.NetworkNamespace,
})
assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.UTSNamespace,
})
assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.PIDNamespace,
})
assert.NotContains(t, spec.Linux.Namespaces, runtimespec.LinuxNamespace{
Type: runtimespec.IPCNamespace,
})
assert.NotContains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "0")
assert.NotContains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "0 2147483647")
},
},
"should set supplemental groups correctly": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
SupplementalGroups: []int64{1111, 2222},
}
},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
require.NotNil(t, spec.Process)
assert.Contains(t, spec.Process.User.AdditionalGids, uint32(1111))
assert.Contains(t, spec.Process.User.AdditionalGids, uint32(2222))
},
},
"should overwrite default sysctls": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.Sysctls = map[string]string{
"net.ipv4.ip_unprivileged_port_start": "500",
"net.ipv4.ping_group_range": "1 1000",
}
},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
require.NotNil(t, spec.Process)
assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ip_unprivileged_port_start"], "500")
assert.Contains(t, spec.Linux.Sysctl["net.ipv4.ping_group_range"], "1 1000")
},
},
"sandbox sizing annotations should be set if LinuxContainerResources were provided": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.Resources = &v1.LinuxContainerResources{
CpuPeriod: 100,
CpuQuota: 200,
CpuShares: 5000,
MemoryLimitInBytes: 1024,
}
},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
value, ok := spec.Annotations[annotations.SandboxCPUPeriod]
assert.True(t, ok)
assert.EqualValues(t, strconv.FormatInt(100, 10), value)
assert.EqualValues(t, "100", value)
value, ok = spec.Annotations[annotations.SandboxCPUQuota]
assert.True(t, ok)
assert.EqualValues(t, "200", value)
value, ok = spec.Annotations[annotations.SandboxCPUShares]
assert.True(t, ok)
assert.EqualValues(t, "5000", value)
value, ok = spec.Annotations[annotations.SandboxMem]
assert.True(t, ok)
assert.EqualValues(t, "1024", value)
},
},
"sandbox sizing annotations should not be set if LinuxContainerResources were not provided": {
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
_, ok := spec.Annotations[annotations.SandboxCPUPeriod]
assert.False(t, ok)
_, ok = spec.Annotations[annotations.SandboxCPUQuota]
assert.False(t, ok)
_, ok = spec.Annotations[annotations.SandboxCPUShares]
assert.False(t, ok)
_, ok = spec.Annotations[annotations.SandboxMem]
assert.False(t, ok)
},
},
"sandbox sizing annotations are zero if the resources are set to 0": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Linux.Resources = &v1.LinuxContainerResources{}
},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
value, ok := spec.Annotations[annotations.SandboxCPUPeriod]
assert.True(t, ok)
assert.EqualValues(t, "0", value)
value, ok = spec.Annotations[annotations.SandboxCPUQuota]
assert.True(t, ok)
assert.EqualValues(t, "0", value)
value, ok = spec.Annotations[annotations.SandboxCPUShares]
assert.True(t, ok)
assert.EqualValues(t, "0", value)
value, ok = spec.Annotations[annotations.SandboxMem]
assert.True(t, ok)
assert.EqualValues(t, "0", value)
},
},
} {
t.Run(desc, func(t *testing.T) {
c := newTestCRIService()
c.config.EnableUnprivilegedICMP = true
c.config.EnableUnprivilegedPorts = true
config, imageConfig, specCheck := getRunPodSandboxTestData()
if test.configChange != nil {
test.configChange(config)
}
spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil)
if test.expectErr {
assert.Error(t, err)
assert.Nil(t, spec)
return
}
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, spec)
if test.specCheck != nil {
test.specCheck(t, spec)
}
})
}
}
func TestSetupSandboxFiles(t *testing.T) {
const (
testID = "test-id"
realhostname = "test-real-hostname"
)
for desc, test := range map[string]struct {
dnsConfig *runtime.DNSConfig
hostname string
ipcMode runtime.NamespaceMode
expectedCalls []ostesting.CalledDetail
}{
"should check host /dev/shm existence when ipc mode is NODE": {
ipcMode: runtime.NamespaceMode_NODE,
expectedCalls: []ostesting.CalledDetail{
{
Name: "Hostname",
},
{
Name: "WriteFile",
Arguments: []interface{}{
filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
[]byte(realhostname + "\n"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts",
filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/resolv.conf",
filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
os.FileMode(0644),
},
},
{
Name: "Stat",
Arguments: []interface{}{"/dev/shm"},
},
},
},
"should create new /etc/resolv.conf if DNSOptions is set": {
dnsConfig: &runtime.DNSConfig{
Servers: []string{"8.8.8.8"},
Searches: []string{"114.114.114.114"},
Options: []string{"timeout:1"},
},
ipcMode: runtime.NamespaceMode_NODE,
expectedCalls: []ostesting.CalledDetail{
{
Name: "Hostname",
},
{
Name: "WriteFile",
Arguments: []interface{}{
filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
[]byte(realhostname + "\n"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts",
filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
os.FileMode(0644),
},
},
{
Name: "WriteFile",
Arguments: []interface{}{
filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
[]byte(`search 114.114.114.114
nameserver 8.8.8.8
options timeout:1
`), os.FileMode(0644),
},
},
{
Name: "Stat",
Arguments: []interface{}{"/dev/shm"},
},
},
},
"should create sandbox shm when ipc namespace mode is not NODE": {
ipcMode: runtime.NamespaceMode_POD,
expectedCalls: []ostesting.CalledDetail{
{
Name: "Hostname",
},
{
Name: "WriteFile",
Arguments: []interface{}{
filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
[]byte(realhostname + "\n"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts",
filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/resolv.conf",
filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
os.FileMode(0644),
},
},
{
Name: "MkdirAll",
Arguments: []interface{}{
filepath.Join(testStateDir, sandboxesDir, testID, "shm"),
os.FileMode(0700),
},
},
{
Name: "Mount",
// Ignore arguments which are too complex to check.
},
},
},
"should create /etc/hostname when hostname is set": {
hostname: "test-hostname",
ipcMode: runtime.NamespaceMode_NODE,
expectedCalls: []ostesting.CalledDetail{
{
Name: "WriteFile",
Arguments: []interface{}{
filepath.Join(testRootDir, sandboxesDir, testID, "hostname"),
[]byte("test-hostname\n"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/hosts",
filepath.Join(testRootDir, sandboxesDir, testID, "hosts"),
os.FileMode(0644),
},
},
{
Name: "CopyFile",
Arguments: []interface{}{
"/etc/resolv.conf",
filepath.Join(testRootDir, sandboxesDir, testID, "resolv.conf"),
os.FileMode(0644),
},
},
{
Name: "Stat",
Arguments: []interface{}{"/dev/shm"},
},
},
},
} {
t.Run(desc, func(t *testing.T) {
c := newTestCRIService()
c.os.(*ostesting.FakeOS).HostnameFn = func() (string, error) {
return realhostname, nil
}
cfg := &runtime.PodSandboxConfig{
Hostname: test.hostname,
DnsConfig: test.dnsConfig,
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{
Ipc: test.ipcMode,
},
},
},
}
c.setupSandboxFiles(testID, cfg)
calls := c.os.(*ostesting.FakeOS).GetCalls()
assert.Len(t, calls, len(test.expectedCalls))
for i, expected := range test.expectedCalls {
if expected.Arguments == nil {
// Ignore arguments.
expected.Arguments = calls[i].Arguments
}
assert.Equal(t, expected, calls[i])
}
})
}
}
func TestParseDNSOption(t *testing.T) {
for desc, test := range map[string]struct {
servers []string
searches []string
options []string
expectedContent string
expectErr bool
}{
"empty dns options should return empty content": {},
"non-empty dns options should return correct content": {
servers: []string{"8.8.8.8", "server.google.com"},
searches: []string{"114.114.114.114"},
options: []string{"timeout:1"},
expectedContent: `search 114.114.114.114
nameserver 8.8.8.8
nameserver server.google.com
options timeout:1
`,
},
"expanded dns config should return correct content on modern libc (e.g. glibc 2.26 and above)": {
servers: []string{"8.8.8.8", "server.google.com"},
searches: []string{
"server0.google.com",
"server1.google.com",
"server2.google.com",
"server3.google.com",
"server4.google.com",
"server5.google.com",
"server6.google.com",
},
options: []string{"timeout:1"},
expectedContent: `search server0.google.com server1.google.com server2.google.com server3.google.com server4.google.com server5.google.com server6.google.com
nameserver 8.8.8.8
nameserver server.google.com
options timeout:1
`,
},
} {
t.Run(desc, func(t *testing.T) {
resolvContent, err := parseDNSOptions(test.servers, test.searches, test.options)
if test.expectErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
assert.Equal(t, resolvContent, test.expectedContent)
})
}
}
func TestSandboxDisableCgroup(t *testing.T) {
config, imageConfig, _ := getRunPodSandboxTestData()
c := newTestCRIService()
c.config.DisableCgroup = true
spec, err := c.sandboxContainerSpec("test-id", config, imageConfig, "test-cni", []string{})
require.NoError(t, err)
t.Log("resource limit should not be set")
assert.Nil(t, spec.Linux.Resources.Memory)
assert.Nil(t, spec.Linux.Resources.CPU)
t.Log("cgroup path should be empty")
assert.Empty(t, spec.Linux.CgroupsPath)
}
// TODO(random-liu): [P1] Add unit test for different error cases to make sure
// the function cleans up on error properly.

20
pkg/cri/sbserver/sandbox_run_other.go
View File

@ -21,29 +21,9 @@ package sbserver
import ( import (
"github.com/containerd/containerd" "github.com/containerd/containerd"
"github.com/containerd/containerd/oci"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
) )
func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (_ *runtimespec.Spec, retErr error) {
return c.runtimeSpec(id, "")
}
// sandboxContainerSpecOpts generates OCI spec options for
// the sandbox container.
func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
return []oci.SpecOpts{}, nil
}
// setupSandboxFiles sets up necessary sandbox files including /dev/shm, /etc/hosts,
// /etc/resolv.conf and /etc/hostname.
func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
return nil
}
// cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to // cleanupSandboxFiles unmount some sandbox files, we rely on the removal of sandbox root directory to
// remove these files. Unmount should *NOT* return error if the mount point is already unmounted. // remove these files. Unmount should *NOT* return error if the mount point is already unmounted.
func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {

36
pkg/cri/sbserver/sandbox_run_other_test.go
View File

@ -1,36 +0,0 @@
//go:build !windows && !linux
// +build !windows,!linux
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sbserver
import (
"testing"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
)
func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
config := &runtime.PodSandboxConfig{}
imageConfig := &imagespec.ImageConfig{}
specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
}
return config, imageConfig, specCheck
}

348
pkg/cri/sbserver/sandbox_run_test.go
View File

@ -19,152 +19,13 @@ package sbserver
import ( import (
"context" "context"
"net" "net"
goruntime "runtime"
"testing" "testing"
"github.com/containerd/go-cni" "github.com/containerd/go-cni"
"github.com/containerd/typeurl"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
criconfig "github.com/containerd/containerd/pkg/cri/config"
sandboxstore "github.com/containerd/containerd/pkg/cri/store/sandbox"
) )
func TestSandboxContainerSpec(t *testing.T) {
switch goruntime.GOOS {
case "darwin":
t.Skip("not implemented on Darwin")
case "freebsd":
t.Skip("not implemented on FreeBSD")
}
testID := "test-id"
nsPath := "test-cni"
for desc, test := range map[string]struct {
configChange func(*runtime.PodSandboxConfig)
podAnnotations []string
imageConfigChange func(*imagespec.ImageConfig)
specCheck func(*testing.T, *runtimespec.Spec)
expectErr bool
}{
"should return error when entrypoint and cmd are empty": {
imageConfigChange: func(c *imagespec.ImageConfig) {
c.Entrypoint = nil
c.Cmd = nil
},
expectErr: true,
},
"a passthrough annotation should be passed as an OCI annotation": {
podAnnotations: []string{"c"},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
assert.Equal(t, spec.Annotations["c"], "d")
},
},
"a non-passthrough annotation should not be passed as an OCI annotation": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Annotations["d"] = "e"
},
podAnnotations: []string{"c"},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
assert.Equal(t, spec.Annotations["c"], "d")
_, ok := spec.Annotations["d"]
assert.False(t, ok)
},
},
"passthrough annotations should support wildcard match": {
configChange: func(c *runtime.PodSandboxConfig) {
c.Annotations["t.f"] = "j"
c.Annotations["z.g"] = "o"
c.Annotations["z"] = "o"
c.Annotations["y.ca"] = "b"
c.Annotations["y"] = "b"
},
podAnnotations: []string{"t*", "z.*", "y.c*"},
specCheck: func(t *testing.T, spec *runtimespec.Spec) {
assert.Equal(t, spec.Annotations["t.f"], "j")
assert.Equal(t, spec.Annotations["z.g"], "o")
assert.Equal(t, spec.Annotations["y.ca"], "b")
_, ok := spec.Annotations["y"]
assert.False(t, ok)
_, ok = spec.Annotations["z"]
assert.False(t, ok)
},
},
} {
t.Run(desc, func(t *testing.T) {
c := newTestCRIService()
config, imageConfig, specCheck := getRunPodSandboxTestData()
if test.configChange != nil {
test.configChange(config)
}
if test.imageConfigChange != nil {
test.imageConfigChange(imageConfig)
}
spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath,
test.podAnnotations)
if test.expectErr {
assert.Error(t, err)
assert.Nil(t, spec)
return
}
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, spec)
if test.specCheck != nil {
test.specCheck(t, spec)
}
})
}
}
func TestTypeurlMarshalUnmarshalSandboxMeta(t *testing.T) {
for desc, test := range map[string]struct {
configChange func(*runtime.PodSandboxConfig)
}{
"should marshal original config": {},
"should marshal Linux": {
configChange: func(c *runtime.PodSandboxConfig) {
if c.Linux == nil {
c.Linux = &runtime.LinuxPodSandboxConfig{}
}
c.Linux.SecurityContext = &runtime.LinuxSandboxSecurityContext{
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
SupplementalGroups: []int64{1111, 2222},
}
},
},
} {
t.Run(desc, func(t *testing.T) {
meta := &sandboxstore.Metadata{
ID: "1",
Name: "sandbox_1",
NetNSPath: "/home/cloud",
}
meta.Config, _, _ = getRunPodSandboxTestData()
if test.configChange != nil {
test.configChange(meta.Config)
}
any, err := typeurl.MarshalAny(meta)
assert.NoError(t, err)
data, err := typeurl.UnmarshalAny(any)
assert.NoError(t, err)
assert.IsType(t, &sandboxstore.Metadata{}, data)
curMeta, ok := data.(*sandboxstore.Metadata)
assert.True(t, ok)
assert.Equal(t, meta, curMeta)
})
}
}
func TestToCNIPortMappings(t *testing.T) { func TestToCNIPortMappings(t *testing.T) {
for desc, test := range map[string]struct { for desc, test := range map[string]struct {
criPortMappings []*runtime.PortMapping criPortMappings []*runtime.PortMapping
@ -315,212 +176,3 @@ func TestSelectPodIP(t *testing.T) {
}) })
} }
} }
func TestHostAccessingSandbox(t *testing.T) {
privilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: true,
},
},
}
nonPrivilegedContext := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
},
},
}
hostNamespace := &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
},
},
}
tests := []struct {
name string
config *runtime.PodSandboxConfig
want bool
}{
{"Security Context is nil", nil, false},
{"Security Context is privileged", privilegedContext, false},
{"Security Context is not privileged", nonPrivilegedContext, false},
{"Security Context namespace host access", hostNamespace, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := hostAccessingSandbox(tt.config); got != tt.want {
t.Errorf("hostAccessingSandbox() = %v, want %v", got, tt.want)
}
})
}
}
func TestGetSandboxRuntime(t *testing.T) {
untrustedWorkloadRuntime := criconfig.Runtime{
Type: "io.containerd.runtime.v1.linux",
Engine: "untrusted-workload-runtime",
Root: "",
}
defaultRuntime := criconfig.Runtime{
Type: "io.containerd.runtime.v1.linux",
Engine: "default-runtime",
Root: "",
}
fooRuntime := criconfig.Runtime{
Type: "io.containerd.runtime.v1.linux",
Engine: "foo-bar",
Root: "",
}
for desc, test := range map[string]struct {
sandboxConfig *runtime.PodSandboxConfig
runtimeHandler string
runtimes map[string]criconfig.Runtime
expectErr bool
expectedRuntime criconfig.Runtime
}{
"should return error if untrusted workload requires host access": {
sandboxConfig: &runtime.PodSandboxConfig{
Linux: &runtime.LinuxPodSandboxConfig{
SecurityContext: &runtime.LinuxSandboxSecurityContext{
Privileged: false,
NamespaceOptions: &runtime.NamespaceOption{
Network: runtime.NamespaceMode_NODE,
Pid: runtime.NamespaceMode_NODE,
Ipc: runtime.NamespaceMode_NODE,
},
},
},
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
},
expectErr: true,
},
"should use untrusted workload runtime for untrusted workload": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
},
expectedRuntime: untrustedWorkloadRuntime,
},
"should use default runtime for regular workload": {
sandboxConfig: &runtime.PodSandboxConfig{},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
},
expectedRuntime: defaultRuntime,
},
"should use default runtime for trusted workload": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "false",
},
},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
},
expectedRuntime: defaultRuntime,
},
"should return error if untrusted workload runtime is required but not configured": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
},
expectErr: true,
},
"should use 'untrusted' runtime for untrusted workload": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
},
expectedRuntime: untrustedWorkloadRuntime,
},
"should use 'untrusted' runtime for untrusted workload & handler": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimeHandler: "untrusted",
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
},
expectedRuntime: untrustedWorkloadRuntime,
},
"should return an error if untrusted annotation with conflicting handler": {
sandboxConfig: &runtime.PodSandboxConfig{
Annotations: map[string]string{
annotations.UntrustedWorkload: "true",
},
},
runtimeHandler: "foo",
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
"foo": fooRuntime,
},
expectErr: true,
},
"should use correct runtime for a runtime handler": {
sandboxConfig: &runtime.PodSandboxConfig{},
runtimeHandler: "foo",
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
criconfig.RuntimeUntrusted: untrustedWorkloadRuntime,
"foo": fooRuntime,
},
expectedRuntime: fooRuntime,
},
"should return error if runtime handler is required but not configured": {
sandboxConfig: &runtime.PodSandboxConfig{},
runtimeHandler: "bar",
runtimes: map[string]criconfig.Runtime{
criconfig.RuntimeDefault: defaultRuntime,
"foo": fooRuntime,
},
expectErr: true,
},
} {
t.Run(desc, func(t *testing.T) {
cri := newTestCRIService()
cri.config = criconfig.Config{
PluginConfig: criconfig.DefaultConfig(),
}
cri.config.ContainerdConfig.DefaultRuntimeName = criconfig.RuntimeDefault
cri.config.ContainerdConfig.Runtimes = test.runtimes
r, err := cri.getSandboxRuntime(test.sandboxConfig, test.runtimeHandler)
assert.Equal(t, test.expectErr, err != nil)
assert.Equal(t, test.expectedRuntime, r)
})
}
}

81
pkg/cri/sbserver/sandbox_run_windows.go
View File

@ -17,91 +17,10 @@
package sbserver package sbserver
import ( import (
"fmt"
"strconv"
"github.com/containerd/containerd" "github.com/containerd/containerd"
"github.com/containerd/containerd/oci"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1" runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
customopts "github.com/containerd/containerd/pkg/cri/opts"
) )
func (c *criService) sandboxContainerSpec(id string, config *runtime.PodSandboxConfig,
imageConfig *imagespec.ImageConfig, nsPath string, runtimePodAnnotations []string) (*runtimespec.Spec, error) {
// Creates a spec Generator with the default spec.
specOpts := []oci.SpecOpts{
oci.WithEnv(imageConfig.Env),
oci.WithHostname(config.GetHostname()),
}
if imageConfig.WorkingDir != "" {
specOpts = append(specOpts, oci.WithProcessCwd(imageConfig.WorkingDir))
}
if len(imageConfig.Entrypoint) == 0 && len(imageConfig.Cmd) == 0 {
// Pause image must have entrypoint or cmd.
return nil, fmt.Errorf("invalid empty entrypoint and cmd in image config %+v", imageConfig)
}
specOpts = append(specOpts, oci.WithProcessArgs(append(imageConfig.Entrypoint, imageConfig.Cmd...)...))
specOpts = append(specOpts,
// Clear the root location since hcsshim expects it.
// NOTE: readonly rootfs doesn't work on windows.
customopts.WithoutRoot,
customopts.WithWindowsNetworkNamespace(nsPath),
)
specOpts = append(specOpts, customopts.WithWindowsDefaultSandboxShares)
// Start with the image config user and override below if RunAsUsername is not "".
username := imageConfig.User
runAsUser := config.GetWindows().GetSecurityContext().GetRunAsUsername()
if runAsUser != "" {
username = runAsUser
}
cs := config.GetWindows().GetSecurityContext().GetCredentialSpec()
if cs != "" {
specOpts = append(specOpts, customopts.WithWindowsCredentialSpec(cs))
}
// There really isn't a good Windows way to verify that the username is available in the
// image as early as here like there is for Linux. Later on in the stack hcsshim
// will handle the behavior of erroring out if the user isn't available in the image
// when trying to run the init process.
specOpts = append(specOpts, oci.WithUser(username))
for pKey, pValue := range getPassthroughAnnotations(config.Annotations,
runtimePodAnnotations) {
specOpts = append(specOpts, customopts.WithAnnotation(pKey, pValue))
}
specOpts = append(specOpts,
customopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox),
customopts.WithAnnotation(annotations.SandboxID, id),
customopts.WithAnnotation(annotations.SandboxNamespace, config.GetMetadata().GetNamespace()),
customopts.WithAnnotation(annotations.SandboxName, config.GetMetadata().GetName()),
customopts.WithAnnotation(annotations.SandboxLogDir, config.GetLogDirectory()),
customopts.WithAnnotation(annotations.WindowsHostProcess, strconv.FormatBool(config.GetWindows().GetSecurityContext().GetHostProcess())),
)
return c.runtimeSpec(id, "", specOpts...)
}
// No sandbox container spec options for windows yet.
func (c *criService) sandboxContainerSpecOpts(config *runtime.PodSandboxConfig, imageConfig *imagespec.ImageConfig) ([]oci.SpecOpts, error) {
return nil, nil
}
// No sandbox files needed for windows.
func (c *criService) setupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
return nil
}
// No sandbox files needed for windows. // No sandbox files needed for windows.
func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error { func (c *criService) cleanupSandboxFiles(id string, config *runtime.PodSandboxConfig) error {
return nil return nil

108
pkg/cri/sbserver/sandbox_run_windows_test.go
View File

@ -1,108 +0,0 @@
/*
Copyright The containerd Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package sbserver
import (
"testing"
imagespec "github.com/opencontainers/image-spec/specs-go/v1"
runtimespec "github.com/opencontainers/runtime-spec/specs-go"
"github.com/stretchr/testify/assert"
runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
"github.com/containerd/containerd/pkg/cri/annotations"
"github.com/containerd/containerd/pkg/cri/opts"
)
func getRunPodSandboxTestData() (*runtime.PodSandboxConfig, *imagespec.ImageConfig, func(*testing.T, string, *runtimespec.Spec)) {
config := &runtime.PodSandboxConfig{
Metadata: &runtime.PodSandboxMetadata{
Name: "test-name",
Uid: "test-uid",
Namespace: "test-ns",
Attempt: 1,
},
Hostname: "test-hostname",
LogDirectory: "test-log-directory",
Labels: map[string]string{"a": "b"},
Annotations: map[string]string{"c": "d"},
Windows: &runtime.WindowsPodSandboxConfig{
SecurityContext: &runtime.WindowsSandboxSecurityContext{
RunAsUsername: "test-user",
CredentialSpec: "{\"test\": \"spec\"}",
HostProcess: false,
},
},
}
imageConfig := &imagespec.ImageConfig{
Env: []string{"a=b", "c=d"},
Entrypoint: []string{"/pause"},
Cmd: []string{"forever"},
WorkingDir: "/workspace",
User: "test-image-user",
}
specCheck := func(t *testing.T, id string, spec *runtimespec.Spec) {
assert.Equal(t, "test-hostname", spec.Hostname)
assert.Nil(t, spec.Root)
assert.Contains(t, spec.Process.Env, "a=b", "c=d")
assert.Equal(t, []string{"/pause", "forever"}, spec.Process.Args)
assert.Equal(t, "/workspace", spec.Process.Cwd)
assert.EqualValues(t, *spec.Windows.Resources.CPU.Shares, opts.DefaultSandboxCPUshares)
// Also checks if override of the image configs user is behaving.
t.Logf("Check username")
assert.Contains(t, spec.Process.User.Username, "test-user")
t.Logf("Check credential spec")
assert.Contains(t, spec.Windows.CredentialSpec, "{\"test\": \"spec\"}")
t.Logf("Check PodSandbox annotations")
assert.Contains(t, spec.Annotations, annotations.SandboxID)
assert.EqualValues(t, spec.Annotations[annotations.SandboxID], id)
assert.Contains(t, spec.Annotations, annotations.ContainerType)
assert.EqualValues(t, spec.Annotations[annotations.ContainerType], annotations.ContainerTypeSandbox)
assert.Contains(t, spec.Annotations, annotations.SandboxNamespace)
assert.EqualValues(t, spec.Annotations[annotations.SandboxNamespace], "test-ns")
assert.Contains(t, spec.Annotations, annotations.SandboxName)
assert.EqualValues(t, spec.Annotations[annotations.SandboxName], "test-name")
assert.Contains(t, spec.Annotations, annotations.SandboxLogDir)
assert.EqualValues(t, spec.Annotations[annotations.SandboxLogDir], "test-log-directory")
assert.Contains(t, spec.Annotations, annotations.WindowsHostProcess)
assert.EqualValues(t, spec.Annotations[annotations.WindowsHostProcess], "false")
}
return config, imageConfig, specCheck
}
func TestSandboxWindowsNetworkNamespace(t *testing.T) {
testID := "test-id"
nsPath := "test-cni"
c := newTestCRIService()
config, imageConfig, specCheck := getRunPodSandboxTestData()
spec, err := c.sandboxContainerSpec(testID, config, imageConfig, nsPath, nil)
assert.NoError(t, err)
assert.NotNil(t, spec)
specCheck(t, testID, spec)
assert.NotNil(t, spec.Windows)
assert.NotNil(t, spec.Windows.Network)
assert.Equal(t, nsPath, spec.Windows.Network.NetworkNamespace)
}